From cb47480300c9552f79eefd7d018495451c09c3f8 Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 17 Aug 2023 03:40:19 -0700 Subject: [PATCH 01/29] Implement ipni-sync http over libp2p Use the new libp2phttp functionality for serving and requesting ipnisync over libp2p. --- dagsync/ipnisync/publisher.go | 33 ++++++- dagsync/ipnisync/publisher_test.go | 149 +++++++++++++++++++++++++++++ dagsync/ipnisync/sync.go | 83 ++++++++++++---- go.mod | 2 +- go.sum | 4 +- 5 files changed, 245 insertions(+), 26 deletions(-) diff --git a/dagsync/ipnisync/publisher.go b/dagsync/ipnisync/publisher.go index 92c060b..8b2338d 100644 --- a/dagsync/ipnisync/publisher.go +++ b/dagsync/ipnisync/publisher.go @@ -60,12 +60,12 @@ func NewPublisher(address string, lsys ipld.LinkSystem, privKey ic.PrivKey, opti return nil, err } proto = multiaddr.Join(proto, httpath) - handlerPath = "/" + handlerPath + handlerPath = handlerPath } pub := &Publisher{ lsys: lsys, - handlerPath: path.Join(handlerPath, IpniPath), + handlerPath: strings.TrimPrefix(path.Join(handlerPath, IpniPath), "/"), peerID: peerID, privKey: privKey, topic: opts.topic, @@ -126,6 +126,27 @@ func NewPublisherWithoutServer(address, handlerPath string, lsys ipld.LinkSystem return NewPublisher(address, lsys, privKey, WithHandlerPath(handlerPath), WithServer(false)) } +// NewPublisherHandler returns a Publisher for use as an http.Handler. Does not +// listen or know about a url prefix. +func NewPublisherHandler(lsys ipld.LinkSystem, privKey ic.PrivKey) (*Publisher, error) { + if privKey == nil { + return nil, errors.New("private key required to sign head requests") + } + peerID, err := peer.IDFromPrivateKey(privKey) + if err != nil { + return nil, fmt.Errorf("could not get peer id from private key: %w", err) + } + + return &Publisher{ + addr: nil, + closer: io.NopCloser(nil), + lsys: lsys, + handlerPath: strings.TrimPrefix(IpniPath, "/"), + peerID: peerID, + privKey: privKey, + }, nil +} + // Addrs returns the addresses, as []multiaddress, that the Publisher is // listening on. func (p *Publisher) Addrs() []multiaddr.Multiaddr { @@ -157,12 +178,14 @@ func (p *Publisher) Close() error { // ServeHTTP implements the http.Handler interface. func (p *Publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if p.handlerPath != "" && !strings.HasPrefix(r.URL.Path, p.handlerPath) { - http.Error(w, "invalid request path: "+r.URL.Path, http.StatusBadRequest) + // A URL path from http will have a leading "/". A URL from libp2phttp will not. + urlPath := strings.TrimPrefix(r.URL.Path, "/") + if p.handlerPath != "" && !strings.HasPrefix(urlPath, p.handlerPath) { + http.Error(w, "invalid request path: "+urlPath, http.StatusBadRequest) return } - ask := path.Base(r.URL.Path) + ask := path.Base(urlPath) if ask == "head" { // serve the head p.lock.Lock() diff --git a/dagsync/ipnisync/publisher_test.go b/dagsync/ipnisync/publisher_test.go index e83f9d1..bb24774 100644 --- a/dagsync/ipnisync/publisher_test.go +++ b/dagsync/ipnisync/publisher_test.go @@ -16,19 +16,168 @@ import ( "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/storage/memstore" "github.com/ipld/go-ipld-prime/traversal" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "github.com/ipni/go-libipni/announce" "github.com/ipni/go-libipni/announce/message" "github.com/ipni/go-libipni/dagsync/ipnisync" + "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + libp2phttp "github.com/libp2p/go-libp2p/p2p/http" "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-multicodec" "github.com/stretchr/testify/require" ) +func TestPublisherWithLibp2pHTTP(t *testing.T) { + ctx := context.Background() + req := require.New(t) + + publisherStore := &correctedMemStore{&memstore.Store{ + Bag: make(map[string][]byte), + }} + publisherLsys := cidlink.DefaultLinkSystem() + publisherLsys.TrustedStorage = true + publisherLsys.SetReadStorage(publisherStore) + publisherLsys.SetWriteStorage(publisherStore) + + privKey, _, err := crypto.GenerateKeyPairWithReader(crypto.Ed25519, 256, rand.Reader) + req.NoError(err) + + publisher, err := ipnisync.NewPublisherHandler(publisherLsys, privKey) + req.NoError(err) + + // Use same identity as publisher. This is necessary so that same ID that + // the publisher uses to sign head/ query responses is the same as the ID + // used to identify the publisherStreamHost. Otherwise, it would be + // necessary for the sync client to know both IDs: one for the stream host + // to connect to, and one for the publisher to validate the dignatuse with. + publisherStreamHost, err := libp2p.New(libp2p.Identity(privKey), libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0")) + req.NoError(err) + + // This is the "HTTP Host". It's like the libp2p "stream host" (aka core + // host.Host), but it uses HTTP semantics instead of stream semantics. + // + // You can pass in options on creation like a stream host to do HTTP over + // libp2p streams, and multiaddrs to create listeners on. + publisherHost := &libp2phttp.HTTPHost{ + StreamHost: publisherStreamHost, + // In production this must be an https endpoint. + ListenAddrs: []multiaddr.Multiaddr{multiaddr.StringCast("/ip4/127.0.0.1/tcp/0/http")}, + ServeInsecureHTTP: true, + } + + // Here is where we attach our request handler. This mounts the + // "/ipnisync/v1" protocol at "/ipni/v1/ad/". If the path "/foo/" was + // given, this would mount it at "/foo/ipni/v1/ad/". libp2phttp manages + // this mapping and clients can learn about the mapping at + // .well-known/libp2p. + // + // In this case we also want out HTTP handler to not even know about the + // prefix, so we use the stdlib http.StripPrefix. + publisherHost.SetHTTPHandlerAtPath(ipnisync.ProtocolID, "/", publisher) + + go publisherHost.Serve() + defer publisherHost.Close() + + serverStreamMa := publisherHost.Addrs()[0] + serverHTTPMa := publisherHost.Addrs()[1] + req.Contains(serverHTTPMa.String(), "/http") + t.Log("libp2p stream server address:", serverStreamMa.String()) + t.Log("libp2p http server address:", serverHTTPMa.String()) + + link, err := publisherLsys.Store( + ipld.LinkContext{Ctx: ctx}, + cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: uint64(multicodec.DagJson), + MhType: uint64(multicodec.Sha2_256), + MhLength: -1, + }, + }, + fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { + na.AssembleEntry("fish").AssignString("lobster") + na.AssembleEntry("fish1").AssignString("lobster1") + na.AssembleEntry("fish2").AssignString("lobster2") + na.AssembleEntry("fish0").AssignString("lobster0") + })) + req.NoError(err) + publisher.SetRoot(link.(cidlink.Link).Cid) + + testCases := []struct { + name string + publisher peer.AddrInfo + newClientHost func(t *testing.T) *libp2phttp.HTTPHost + }{ + { + "HTTP transport", + peer.AddrInfo{Addrs: []multiaddr.Multiaddr{serverHTTPMa}}, + func(t *testing.T) *libp2phttp.HTTPHost { + return &libp2phttp.HTTPHost{} + }, + }, + { + "libp2p stream transport", + peer.AddrInfo{ID: publisherStreamHost.ID(), Addrs: []multiaddr.Multiaddr{serverStreamMa}}, + func(t *testing.T) *libp2phttp.HTTPHost { + clientStreamHost, err := libp2p.New(libp2p.NoListenAddrs) + req.NoError(err) + return &libp2phttp.HTTPHost{ + StreamHost: clientStreamHost, + } + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Plumbing to set up the test. + clientStore := &correctedMemStore{&memstore.Store{ + Bag: make(map[string][]byte), + }} + clientLsys := cidlink.DefaultLinkSystem() + clientLsys.TrustedStorage = true + clientLsys.SetReadStorage(clientStore) + clientLsys.SetWriteStorage(clientStore) + clientSync := ipnisync.NewLibp2pSync(clientLsys, tc.newClientHost(t), nil) + + // In a dagsync Subscriber, the clientSync is created once and + // lives for the lifetime of the Subscriber (lifetime of indexer), + // The clientSyncer is created for each sync operation and only + // lives for the duration of the sync. The publisher's address may + // change from one sync to the next, and we do not know the + // addresses ahead of time. + clientSyncer, err := clientSync.NewSyncer(tc.publisher.ID, tc.publisher.Addrs) + req.NoError(err) + + headCid, err := clientSyncer.GetHead(ctx) + req.NoError(err) + + req.Equal(link.(cidlink.Link).Cid, headCid) + + clientSyncer.Sync(ctx, headCid, selectorparse.CommonSelector_MatchPoint) + require.NoError(t, err) + + // Assert that data is loadable from the link system. + wantLink := cidlink.Link{Cid: headCid} + node, err := clientLsys.Load(ipld.LinkContext{Ctx: ctx}, wantLink, basicnode.Prototype.Any) + require.NoError(t, err) + + // Assert synced node link matches the computed link, i.e. is spec-compliant. + gotLink, err := clientLsys.ComputeLink(wantLink.Prototype(), node) + require.NoError(t, err) + require.Equal(t, gotLink, wantLink, "computed %s but got %s", gotLink.String(), wantLink.String()) + }) + } +} + func TestNewPublisherForListener(t *testing.T) { req := require.New(t) ctx := context.Background() diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index e984606..5950e36 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -21,19 +21,37 @@ import ( headschema "github.com/ipni/go-libipni/dagsync/ipnisync/head" "github.com/ipni/go-libipni/maurl" "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" + libp2phttp "github.com/libp2p/go-libp2p/p2p/http" "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multihash" ) +const ProtocolID = protocol.ID("/ipnisync/v1") + const defaultHttpTimeout = 10 * time.Second var log = logging.Logger("dagsync/ipnisync") +var errHeadFromUnexpectedPeer = errors.New("found head signed from an unexpected peer") + // Sync provides sync functionality for use with all http syncs. type Sync struct { blockHook func(peer.ID, cid.Cid) client *http.Client lsys ipld.LinkSystem + + // libp2phttp + clientHost *libp2phttp.HTTPHost +} + +// Syncer provides sync functionality for a single sync with a peer. +type Syncer struct { + client *http.Client + peerID peer.ID + rootURL url.URL + urls []*url.URL + sync *Sync } // NewSync creates a new Sync. @@ -50,11 +68,47 @@ func NewSync(lsys ipld.LinkSystem, client *http.Client, blockHook func(peer.ID, } } -// NewSyncer creates a new Syncer to use for a single sync operation against a peer. +func NewLibp2pSync(lsys ipld.LinkSystem, clientHost *libp2phttp.HTTPHost, blockHook func(peer.ID, cid.Cid)) *Sync { + return &Sync{ + blockHook: blockHook, + lsys: lsys, + + clientHost: clientHost, + } +} + +// NewSyncer creates a new Syncer to use for a single sync operation against a +// peer. A value for peerID is optional for the HTTP transport. func (s *Sync) NewSyncer(peerID peer.ID, peerAddrs []multiaddr.Multiaddr) (*Syncer, error) { - urls := make([]*url.URL, len(peerAddrs)) - for i := range peerAddrs { - u, err := maurl.ToURL(peerAddrs[i]) + peerInfo := peer.AddrInfo{ + ID: peerID, + Addrs: peerAddrs, + } + if s.clientHost != nil { + return s.newLibp2pSyncer(peerInfo) + } + return s.newSyncer(peerInfo) +} + +func (s *Sync) newLibp2pSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { + httpClient, err := s.clientHost.NamespacedClient(ProtocolID, peerInfo) + if err != nil { + return nil, err + } + + return &Syncer{ + client: &httpClient, + peerID: peerInfo.ID, + rootURL: url.URL{Path: IpniPath}, + urls: nil, + sync: s, + }, nil +} + +func (s *Sync) newSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { + urls := make([]*url.URL, len(peerInfo.Addrs)) + for i, addr := range peerInfo.Addrs { + u, err := maurl.ToURL(addr) if err != nil { return nil, err } @@ -62,7 +116,8 @@ func (s *Sync) NewSyncer(peerID peer.ID, peerAddrs []multiaddr.Multiaddr) (*Sync } return &Syncer{ - peerID: peerID, + client: s.client, + peerID: peerInfo.ID, rootURL: *urls[0], urls: urls[1:], sync: s, @@ -73,16 +128,6 @@ func (s *Sync) Close() { s.client.CloseIdleConnections() } -var errHeadFromUnexpectedPeer = errors.New("found head signed from an unexpected peer") - -// Syncer provides sync functionality for a single sync with a peer. -type Syncer struct { - peerID peer.ID - rootURL url.URL - urls []*url.URL - sync *Sync -} - // GetHead fetches the head of the peer's advertisement chain. func (s *Syncer) GetHead(ctx context.Context) (cid.Cid, error) { var signedHead *headschema.SignedHead @@ -99,7 +144,9 @@ func (s *Syncer) GetHead(ctx context.Context) (cid.Cid, error) { if err != nil { return cid.Undef, err } - if signerID != s.peerID { + if s.peerID == "" { + log.Warn("cannot verify publisher signature without peer ID") + } else if signerID != s.peerID { return cid.Undef, errHeadFromUnexpectedPeer } @@ -138,7 +185,7 @@ func (s *Syncer) Sync(ctx context.Context, nextCid cid.Cid, sel ipld.Node) error } } - s.sync.client.CloseIdleConnections() + s.client.CloseIdleConnections() return nil } @@ -207,7 +254,7 @@ nextURL: return err } - resp, err := s.sync.client.Do(req) + resp, err := s.client.Do(req) if err != nil { if len(s.urls) != 0 { log.Errorw("Fetch request failed, will retry with next address", "err", err) diff --git a/go.mod b/go.mod index f094f84..a02375a 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/ipfs/go-ipld-format v0.3.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-ipld-prime v0.20.0 - github.com/libp2p/go-libp2p v0.29.2 + github.com/libp2p/go-libp2p v0.29.1-0.20230816203758-9566227604f8 github.com/libp2p/go-libp2p-gostream v0.6.0 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-msgio v0.3.0 diff --git a/go.sum b/go.sum index db430d3..4abe12a 100644 --- a/go.sum +++ b/go.sum @@ -299,8 +299,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.29.2 h1:uPw/c8hOxoLP/KhFnzlc5Ejqf+OmAL1dwIsqE31WBtY= -github.com/libp2p/go-libp2p v0.29.2/go.mod h1:OU7nSq0aEZMsV2wY8nXn1+XNNt9q2UiR8LjW3Kmp2UE= +github.com/libp2p/go-libp2p v0.29.1-0.20230816203758-9566227604f8 h1:nR6nNb5b/f4AF2VdDHZBzWsi/5Gppb6raiHwyKNXKd0= +github.com/libp2p/go-libp2p v0.29.1-0.20230816203758-9566227604f8/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU= From 58a980e1c3e459dcec2a26d12354c35efebe3383 Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 18 Aug 2023 03:37:29 -0700 Subject: [PATCH 02/29] Move libp2phttp functionality inside ipnisync --- dagsync/announce_test.go | 2 +- dagsync/http_test.go | 2 +- dagsync/ipnisync/option.go | 77 +++++++- dagsync/ipnisync/publisher.go | 159 ++++++++++----- dagsync/ipnisync/publisher_test.go | 298 ++++++++++++++++++++++++----- dagsync/ipnisync/sync.go | 85 ++++---- dagsync/ipnisync/sync_test.go | 28 ++- dagsync/subscriber.go | 21 +- dagsync/subscriber_test.go | 2 +- 9 files changed, 521 insertions(+), 153 deletions(-) diff --git a/dagsync/announce_test.go b/dagsync/announce_test.go index e3b6de3..7313a6f 100644 --- a/dagsync/announce_test.go +++ b/dagsync/announce_test.go @@ -154,7 +154,7 @@ func TestAnnounce_LearnsHttpPublisherAddr(t *testing.T) { pubh := test.MkTestHost(t) pubds := dssync.MutexWrap(datastore.NewMapDatastore()) publs := test.MkLinkSystem(pubds) - pub, err := ipnisync.NewPublisher("0.0.0.0:0", publs, pubh.Peerstore().PrivKey(pubh.ID())) + pub, err := ipnisync.NewPublisher(publs, pubh.Peerstore().PrivKey(pubh.ID()), ipnisync.WithHTTPListenAddrs("0.0.0.0:0")) require.NoError(t, err) defer pub.Close() diff --git a/dagsync/http_test.go b/dagsync/http_test.go index cdf08ba..84a561d 100644 --- a/dagsync/http_test.go +++ b/dagsync/http_test.go @@ -40,7 +40,7 @@ func setupPublisherSubscriber(t *testing.T, subscriberOptions []dagsync.Option) srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) srcLinkSys := test.MkLinkSystem(srcStore) - pub, err := ipnisync.NewPublisher("127.0.0.1:0", srcLinkSys, srcPrivKey, ipnisync.WithServer(true)) + pub, err := ipnisync.NewPublisher(srcLinkSys, srcPrivKey, ipnisync.WithHTTPListenAddrs("127.0.0.1:0")) require.NoError(t, err) t.Cleanup(func() { pub.Close() diff --git a/dagsync/ipnisync/option.go b/dagsync/ipnisync/option.go index 9274ca3..ae10945 100644 --- a/dagsync/ipnisync/option.go +++ b/dagsync/ipnisync/option.go @@ -2,6 +2,9 @@ package ipnisync import ( "fmt" + "time" + + "github.com/libp2p/go-libp2p/core/host" ) // pubConfig contains all options for configuring Publisher. @@ -9,6 +12,10 @@ type config struct { handlerPath string startServer bool topic string + + streamHost host.Host + requireTLS bool + httpAddrs []string } // Option is a function that sets a value in a config. @@ -27,6 +34,13 @@ func getOpts(opts []Option) (config, error) { return cfg, nil } +func WithHTTPListenAddrs(addrs ...string) Option { + return func(c *config) error { + c.httpAddrs = append(c.httpAddrs, addrs...) + return nil + } +} + // WithHandlerPath sets the path used to handle requests to this publisher. // This should only include the path before the /ipni/v1/ad/ part of the path. func WithHandlerPath(urlPath string) Option { @@ -45,12 +59,65 @@ func WithHeadTopic(topic string) Option { } } -// WithServer, if true, starts an http server listening on the given address. -// an HTTP server. If this option is not specified, then no server is started -// and this will need to be done by the caller. -func WithServer(serve bool) Option { +// WithStartServer, if true, starts an http server listening on the given +// address. an HTTP server. If this option is not specified, then no server is +// started and this will need to be done by the caller. +func WithStartServer(start bool) Option { + return func(c *config) error { + c.startServer = start + return nil + } +} + +// WithRequireTLS tells whether to allow the publisher to serve non-secure http +// (false) or to require https (true). Default is false, allowing non-secure +// HTTP. +func WithRequireTLS(require bool) Option { + return func(c *config) error { + c.requireTLS = require + return nil + } +} + +func WithStreamHost(h host.Host) Option { return func(c *config) error { - c.startServer = serve + c.streamHost = h return nil } } + +type clientConfig struct { + authPeerID bool + timeout time.Duration + streamHost host.Host +} + +// Option is a function that sets a value in a config. +type ClientOption func(*clientConfig) + +// getClientOpts creates a pubConfig and applies Options to it. +func getClientOpts(opts []ClientOption) clientConfig { + var cfg clientConfig + for _, opt := range opts { + opt(&cfg) + } + return cfg +} + +func ClientAuthPeerID(require bool) ClientOption { + return func(c *clientConfig) { + c.authPeerID = require + } +} + +func ClientTimeout(to time.Duration) ClientOption { + return func(c *clientConfig) { + c.timeout = to + } +} + +func ClientStreamHost(h host.Host) ClientOption { + return func(c *clientConfig) { + c.streamHost = h + } +} diff --git a/dagsync/ipnisync/publisher.go b/dagsync/ipnisync/publisher.go index 8b2338d..b60ef48 100644 --- a/dagsync/ipnisync/publisher.go +++ b/dagsync/ipnisync/publisher.go @@ -3,7 +3,6 @@ package ipnisync import ( "errors" "fmt" - "io" "net" "net/http" "net/url" @@ -17,16 +16,16 @@ import ( cidlink "github.com/ipld/go-ipld-prime/linking/cid" basicnode "github.com/ipld/go-ipld-prime/node/basic" headschema "github.com/ipni/go-libipni/dagsync/ipnisync/head" + "github.com/ipni/go-libipni/maurl" ic "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" + libp2phttp "github.com/libp2p/go-libp2p/p2p/http" "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" ) // Publisher serves an advertisement chain over HTTP. type Publisher struct { addr multiaddr.Multiaddr - closer io.Closer lsys ipld.LinkSystem handlerPath string peerID peer.ID @@ -34,13 +33,17 @@ type Publisher struct { lock sync.Mutex root cid.Cid topic string + + pubHost *libp2phttp.HTTPHost + // haatAddrs is returned by Addrs when not starting the server. + httpAddrs []multiaddr.Multiaddr } var _ http.Handler = (*Publisher)(nil) -// NewPublisher creates a new http publisher. It is optional with start a server, listening on the specified -// address, using the WithStartServer option. -func NewPublisher(address string, lsys ipld.LinkSystem, privKey ic.PrivKey, options ...Option) (*Publisher, error) { +// NewPublisher creates a new ipni-sync publisher. Optionally, a libp2p stream +// host can be provided to serve HTTP over libp2p. +func NewPublisher(lsys ipld.LinkSystem, privKey ic.PrivKey, options ...Option) (*Publisher, error) { opts, err := getOpts(options) if err != nil { return nil, err @@ -52,55 +55,77 @@ func NewPublisher(address string, lsys ipld.LinkSystem, privKey ic.PrivKey, opti if err != nil { return nil, fmt.Errorf("could not get peer id from private key: %w", err) } - proto, _ := multiaddr.NewMultiaddr("/http") - handlerPath := strings.TrimPrefix(opts.handlerPath, "/") - if handlerPath != "" { - httpath, err := multiaddr.NewComponent("httpath", url.PathEscape(handlerPath)) - if err != nil { - return nil, err - } - proto = multiaddr.Join(proto, httpath) - handlerPath = handlerPath + if opts.streamHost != nil && opts.streamHost.ID() != peerID { + return nil, errors.New("stream host ID must match private key ID") } pub := &Publisher{ lsys: lsys, - handlerPath: strings.TrimPrefix(path.Join(handlerPath, IpniPath), "/"), + handlerPath: strings.TrimPrefix(IpniPath, "/"), peerID: peerID, privKey: privKey, topic: opts.topic, } - var addr net.Addr - if opts.startServer { - l, err := net.Listen("tcp", address) + opts.handlerPath = strings.TrimPrefix(opts.handlerPath, "/") + + if !opts.startServer { + httpListenAddrs, err := httpAddrsToMultiaddrs(opts.httpAddrs, opts.requireTLS, opts.handlerPath) if err != nil { return nil, err } - pub.closer = l - addr = l.Addr() - - // Run service on configured port. - server := &http.Server{ - Handler: pub, - Addr: l.Addr().String(), + if opts.streamHost != nil { + return nil, errors.New("server must be started to serve http over stream host") } - go server.Serve(l) - } else { - addr, err = net.ResolveTCPAddr("tcp", address) - if err != nil { - return nil, err + // If the server is not started, the handlerPath does not get stripped + // from the HTTP request, so leave it as part of the prefix to match in + // the SetveHTTP handler. + if opts.handlerPath != "" { + pub.handlerPath = path.Join(opts.handlerPath, pub.handlerPath) } + pub.httpAddrs = httpListenAddrs + return pub, nil } - maddr, err := manet.FromNetAddr(addr) + httpListenAddrs, err := httpAddrsToMultiaddrs(opts.httpAddrs, opts.requireTLS, "") if err != nil { - if pub.closer != nil { - pub.closer.Close() - } return nil, err } - pub.addr = multiaddr.Join(maddr, proto) + if len(httpListenAddrs) == 0 && opts.streamHost == nil { + return nil, errors.New("at least one http listen address or libp2p stream host is needed") + } + + // This is the "HTTP Host". It's like the libp2p "stream host" (aka core + // host.Host), but it uses HTTP semantics instead of stream semantics. + publisherHost := &libp2phttp.HTTPHost{ + StreamHost: opts.streamHost, + ListenAddrs: httpListenAddrs, + ServeInsecureHTTP: !opts.requireTLS, + } + pub.pubHost = publisherHost + + if opts.handlerPath == "" { + opts.handlerPath = "/" + } else { + if !strings.HasPrefix(opts.handlerPath, "/") { + opts.handlerPath = "/" + opts.handlerPath + } + } + + // Here is where this Publisher is attached as a request handler. This + // mounts the "/ipnisync/v1" protocol at "/ipni/v1/ad/". If + // opts.handlerPath is "/foo/", this mounts it at "/foo/ipni/v1/ad/". + // libp2phttp manages this mapping and clients can learn about the mapping + // at .well-known/libp2p. + // + // In this case we also want the HTTP handler to not even know about the + // prefix, so we use the stdlib http.StripPrefix. + publisherHost.SetHTTPHandlerAtPath(ProtocolID, opts.handlerPath, pub) + + go publisherHost.Serve() + + // Calling publisherHost.Addrs() waits until listeners are ready. + log.Infow("Publisher ready", "listenOn", publisherHost.Addrs()) return pub, nil } @@ -111,9 +136,9 @@ func NewPublisher(address string, lsys ipld.LinkSystem, privKey ic.PrivKey, opti // can be used to handle requests. handlerPath is the path to handle // requests on, e.g. "ipni" for `/ipni/...` requests. // -// DEPRECATED: use NewPublisherWithoutServer(listener.Addr(), ...) +// DEPRECATED: use NewPublisher(lsys, privKey, WithHTTPListenAddrs(listener.Addr().String()), WithHandlerPath(handlerPath), WithStartServer(false)) func NewPublisherForListener(listener net.Listener, handlerPath string, lsys ipld.LinkSystem, privKey ic.PrivKey) (*Publisher, error) { - return NewPublisherWithoutServer(listener.Addr().String(), handlerPath, lsys, privKey) + return NewPublisher(lsys, privKey, WithHTTPListenAddrs(listener.Addr().String()), WithHandlerPath(handlerPath), WithStartServer(false)) } // NewPublisherWithoutServer creates a new http publisher for an existing @@ -121,9 +146,9 @@ func NewPublisherForListener(listener net.Listener, handlerPath string, lsys ipl // the HTTP server is the caller's responsibility. ServeHTTP on the // returned Publisher can be used to handle requests. // -// DEPRECATED: use NewPublisher(address, lsys, privKey, WithHandlerPath(handlerPath)) +// DEPRECATED: use NewPublisher(lsys, privKey, WithHTTPListenAddrs(address), WithHandlerPath(handlerPath), WithStartServer(false)) func NewPublisherWithoutServer(address, handlerPath string, lsys ipld.LinkSystem, privKey ic.PrivKey, options ...Option) (*Publisher, error) { - return NewPublisher(address, lsys, privKey, WithHandlerPath(handlerPath), WithServer(false)) + return NewPublisher(lsys, privKey, WithHTTPListenAddrs(address), WithHandlerPath(handlerPath), WithStartServer(false)) } // NewPublisherHandler returns a Publisher for use as an http.Handler. Does not @@ -139,7 +164,6 @@ func NewPublisherHandler(lsys ipld.LinkSystem, privKey ic.PrivKey) (*Publisher, return &Publisher{ addr: nil, - closer: io.NopCloser(nil), lsys: lsys, handlerPath: strings.TrimPrefix(IpniPath, "/"), peerID: peerID, @@ -149,8 +173,15 @@ func NewPublisherHandler(lsys ipld.LinkSystem, privKey ic.PrivKey) (*Publisher, // Addrs returns the addresses, as []multiaddress, that the Publisher is // listening on. +// +// If the server is not started, WithStartServer(false), then this returns the +// multiaddr versions of the list of addresses given by WithHTTPListenAddrs and +// may not actually be a listening address. func (p *Publisher) Addrs() []multiaddr.Multiaddr { - return []multiaddr.Multiaddr{p.addr} + if p.pubHost == nil { + return p.httpAddrs + } + return p.pubHost.Addrs() } // ID returns the p2p peer ID of the Publisher. @@ -173,7 +204,10 @@ func (p *Publisher) SetRoot(c cid.Cid) { // Close closes the Publisher. func (p *Publisher) Close() error { - return p.closer.Close() + if p.pubHost != nil { + return p.pubHost.Close() + } + return nil } // ServeHTTP implements the http.Handler interface. @@ -236,3 +270,42 @@ func newEncodedSignedHead(rootCid cid.Cid, topic string, privKey ic.PrivKey) ([] } return signedHead.Encode() } + +func httpAddrsToMultiaddrs(httpAddrs []string, requireTLS bool, handlerPath string) ([]multiaddr.Multiaddr, error) { + if len(httpAddrs) == 0 { + return nil, nil + } + + var defaultScheme string + if requireTLS { + defaultScheme = "https://" + } else { + defaultScheme = "http://" + } + maddrs := make([]multiaddr.Multiaddr, len(httpAddrs)) + for i, addr := range httpAddrs { + if !strings.HasPrefix(addr, "https://") && !strings.HasPrefix(addr, "http://") { + addr = defaultScheme + addr + } + u, err := url.Parse(addr) + if err != nil { + return nil, err + } + if handlerPath != "" { + u = u.JoinPath(handlerPath) + } + if requireTLS && u.Scheme == "http" { + log.Warnf("Ignored non-secure HTTP address: %s", addr) + continue + } + maddrs[i], err = maurl.FromURL(u) + if err != nil { + return nil, err + } + } + if len(maddrs) == 0 && requireTLS { + return nil, errors.New("no usable http listen addresses: https required") + } + + return maddrs, nil +} diff --git a/dagsync/ipnisync/publisher_test.go b/dagsync/ipnisync/publisher_test.go index bb24774..83dc6f9 100644 --- a/dagsync/ipnisync/publisher_test.go +++ b/dagsync/ipnisync/publisher_test.go @@ -28,8 +28,8 @@ import ( "github.com/ipni/go-libipni/dagsync/ipnisync" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" - libp2phttp "github.com/libp2p/go-libp2p/p2p/http" "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multicodec" "github.com/stretchr/testify/require" @@ -50,9 +50,6 @@ func TestPublisherWithLibp2pHTTP(t *testing.T) { privKey, _, err := crypto.GenerateKeyPairWithReader(crypto.Ed25519, 256, rand.Reader) req.NoError(err) - publisher, err := ipnisync.NewPublisherHandler(publisherLsys, privKey) - req.NoError(err) - // Use same identity as publisher. This is necessary so that same ID that // the publisher uses to sign head/ query responses is the same as the ID // used to identify the publisherStreamHost. Otherwise, it would be @@ -61,33 +58,16 @@ func TestPublisherWithLibp2pHTTP(t *testing.T) { publisherStreamHost, err := libp2p.New(libp2p.Identity(privKey), libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0")) req.NoError(err) - // This is the "HTTP Host". It's like the libp2p "stream host" (aka core - // host.Host), but it uses HTTP semantics instead of stream semantics. - // - // You can pass in options on creation like a stream host to do HTTP over - // libp2p streams, and multiaddrs to create listeners on. - publisherHost := &libp2phttp.HTTPHost{ - StreamHost: publisherStreamHost, - // In production this must be an https endpoint. - ListenAddrs: []multiaddr.Multiaddr{multiaddr.StringCast("/ip4/127.0.0.1/tcp/0/http")}, - ServeInsecureHTTP: true, - } + publisher, err := ipnisync.NewPublisher(publisherLsys, privKey, + ipnisync.WithHTTPListenAddrs("http://127.0.0.1:0"), + ipnisync.WithStreamHost(publisherStreamHost), + ipnisync.WithRequireTLS(false), + ) + req.NoError(err) - // Here is where we attach our request handler. This mounts the - // "/ipnisync/v1" protocol at "/ipni/v1/ad/". If the path "/foo/" was - // given, this would mount it at "/foo/ipni/v1/ad/". libp2phttp manages - // this mapping and clients can learn about the mapping at - // .well-known/libp2p. - // - // In this case we also want out HTTP handler to not even know about the - // prefix, so we use the stdlib http.StripPrefix. - publisherHost.SetHTTPHandlerAtPath(ipnisync.ProtocolID, "/", publisher) - - go publisherHost.Serve() - defer publisherHost.Close() - - serverStreamMa := publisherHost.Addrs()[0] - serverHTTPMa := publisherHost.Addrs()[1] + req.Equal(2, len(publisher.Addrs())) + serverStreamMa := publisher.Addrs()[0] + serverHTTPMa := publisher.Addrs()[1] req.Contains(serverHTTPMa.String(), "/http") t.Log("libp2p stream server address:", serverStreamMa.String()) t.Log("libp2p http server address:", serverHTTPMa.String()) @@ -112,26 +92,24 @@ func TestPublisherWithLibp2pHTTP(t *testing.T) { publisher.SetRoot(link.(cidlink.Link).Cid) testCases := []struct { - name string - publisher peer.AddrInfo - newClientHost func(t *testing.T) *libp2phttp.HTTPHost + name string + publisher peer.AddrInfo + streamHost func(t *testing.T) host.Host }{ { "HTTP transport", peer.AddrInfo{Addrs: []multiaddr.Multiaddr{serverHTTPMa}}, - func(t *testing.T) *libp2phttp.HTTPHost { - return &libp2phttp.HTTPHost{} + func(t *testing.T) host.Host { + return nil }, }, { "libp2p stream transport", peer.AddrInfo{ID: publisherStreamHost.ID(), Addrs: []multiaddr.Multiaddr{serverStreamMa}}, - func(t *testing.T) *libp2phttp.HTTPHost { - clientStreamHost, err := libp2p.New(libp2p.NoListenAddrs) + func(t *testing.T) host.Host { + streamHost, err := libp2p.New(libp2p.NoListenAddrs) req.NoError(err) - return &libp2phttp.HTTPHost{ - StreamHost: clientStreamHost, - } + return streamHost }, }, } @@ -146,7 +124,7 @@ func TestPublisherWithLibp2pHTTP(t *testing.T) { clientLsys.TrustedStorage = true clientLsys.SetReadStorage(clientStore) clientLsys.SetWriteStorage(clientStore) - clientSync := ipnisync.NewLibp2pSync(clientLsys, tc.newClientHost(t), nil) + clientSync := ipnisync.NewSync(clientLsys, nil, ipnisync.ClientStreamHost(tc.streamHost(t))) // In a dagsync Subscriber, the clientSync is created once and // lives for the lifetime of the Subscriber (lifetime of indexer), @@ -154,7 +132,8 @@ func TestPublisherWithLibp2pHTTP(t *testing.T) { // lives for the duration of the sync. The publisher's address may // change from one sync to the next, and we do not know the // addresses ahead of time. - clientSyncer, err := clientSync.NewSyncer(tc.publisher.ID, tc.publisher.Addrs) + t.Log("Syncing to publisher at:", tc.publisher.Addrs) + clientSyncer, err := clientSync.NewSyncer(tc.publisher) req.NoError(err) headCid, err := clientSyncer.GetHead(ctx) @@ -178,6 +157,93 @@ func TestPublisherWithLibp2pHTTP(t *testing.T) { } } +func TestExistingServerWithPublisher(t *testing.T) { + ctx := context.Background() + req := require.New(t) + + publisherStore := &correctedMemStore{&memstore.Store{ + Bag: make(map[string][]byte), + }} + publisherLsys := cidlink.DefaultLinkSystem() + publisherLsys.TrustedStorage = true + publisherLsys.SetReadStorage(publisherStore) + publisherLsys.SetWriteStorage(publisherStore) + + privKey, _, err := crypto.GenerateKeyPairWithReader(crypto.Ed25519, 256, rand.Reader) + req.NoError(err) + + // Start a server without using libp2phttp to: + // 1. Demonstrate this works without using libp2phttp + // 2. Give example of how an existing listener can be used to server the publisher. + const listenAddress = "127.0.0.1:0" + l, err := net.Listen("tcp", listenAddress) + req.NoError(err) + defer l.Close() + addr := "http://" + l.Addr().String() + publisher, err := ipnisync.NewPublisher(publisherLsys, privKey, ipnisync.WithHTTPListenAddrs(addr), ipnisync.WithStartServer(false)) + req.NoError(err) + go http.Serve(l, publisher) + + serverHTTPMa := publisher.Addrs()[0] + req.Contains(serverHTTPMa.String(), "/http") + t.Log("libp2p http server address:", serverHTTPMa.String()) + + link, err := publisherLsys.Store( + ipld.LinkContext{Ctx: ctx}, + cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: uint64(multicodec.DagJson), + MhType: uint64(multicodec.Sha2_256), + MhLength: -1, + }, + }, + fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { + na.AssembleEntry("fish").AssignString("lobster") + na.AssembleEntry("fish1").AssignString("lobster1") + na.AssembleEntry("fish2").AssignString("lobster2") + na.AssembleEntry("fish0").AssignString("lobster0") + })) + req.NoError(err) + publisher.SetRoot(link.(cidlink.Link).Cid) + + pubInfo := peer.AddrInfo{ + ID: publisher.ID(), + Addrs: []multiaddr.Multiaddr{serverHTTPMa}, + } + + // Plumbing to set up the test. + clientStore := &correctedMemStore{&memstore.Store{ + Bag: make(map[string][]byte), + }} + clientLsys := cidlink.DefaultLinkSystem() + clientLsys.TrustedStorage = true + clientLsys.SetReadStorage(clientStore) + clientLsys.SetWriteStorage(clientStore) + clientSync := ipnisync.NewSync(clientLsys, nil) + + t.Log("Syncing to publisher at:", pubInfo.Addrs) + clientSyncer, err := clientSync.NewSyncer(pubInfo) + req.NoError(err) + + headCid, err := clientSyncer.GetHead(ctx) + req.NoError(err) + req.Equal(link.(cidlink.Link).Cid, headCid) + + clientSyncer.Sync(ctx, headCid, selectorparse.CommonSelector_MatchPoint) + require.NoError(t, err) + + // Assert that data is loadable from the link system. + wantLink := cidlink.Link{Cid: headCid} + node, err := clientLsys.Load(ipld.LinkContext{Ctx: ctx}, wantLink, basicnode.Prototype.Any) + require.NoError(t, err) + + // Assert synced node link matches the computed link, i.e. is spec-compliant. + gotLink, err := clientLsys.ComputeLink(wantLink.Prototype(), node) + require.NoError(t, err) + require.Equal(t, gotLink, wantLink, "computed %s but got %s", gotLink.String(), wantLink.String()) +} + func TestNewPublisherForListener(t *testing.T) { req := require.New(t) ctx := context.Background() @@ -200,7 +266,7 @@ func TestNewPublisherForListener(t *testing.T) { privKey, _, err := crypto.GenerateKeyPairWithReader(crypto.RSA, 2048, rand.Reader) req.NoError(err) sender := &fakeSender{} - subject, err := ipnisync.NewPublisher(l.Addr().String(), lsys, privKey, ipnisync.WithServer(false), ipnisync.WithHandlerPath(handlerPath)) + subject, err := ipnisync.NewPublisher(lsys, privKey, ipnisync.WithHTTPListenAddrs(l.Addr().String()), ipnisync.WithHandlerPath(handlerPath), ipnisync.WithStartServer(false)) req.NoError(err) rootCid := rootLnk.(cidlink.Link).Cid @@ -222,8 +288,7 @@ func TestNewPublisherForListener(t *testing.T) { u := &url.URL{ Path: path.Join("/", handlerPath, ipnisync.IpniPath, "/head"), } - //u = u.JoinPath(handlerPath, ipnisync.IpniPath, "/head") - // + subject.ServeHTTP(resp, &http.Request{URL: u}) req.Equal(0, resp.status) // not explicitly set req.Nil(resp.header) @@ -245,6 +310,149 @@ func TestNewPublisherForListener(t *testing.T) { } } +func TestHandlerPath(t *testing.T) { + //t.Skip("needs work") + req := require.New(t) + ctx := context.Background() + + publisherStore := &correctedMemStore{&memstore.Store{ + Bag: make(map[string][]byte), + }} + publisherLsys := cidlink.DefaultLinkSystem() + publisherLsys.TrustedStorage = true + publisherLsys.SetReadStorage(publisherStore) + publisherLsys.SetWriteStorage(publisherStore) + + privKey, _, err := crypto.GenerateKeyPairWithReader(crypto.Ed25519, 256, rand.Reader) + req.NoError(err) + + publisher, err := ipnisync.NewPublisher(publisherLsys, privKey, + ipnisync.WithHTTPListenAddrs("http://127.0.0.1:0"), + ipnisync.WithHandlerPath("/boop/bop/beep"), + ) + req.NoError(err) + + req.Equal(1, len(publisher.Addrs())) + serverHTTPMa := publisher.Addrs()[0] + req.Contains(serverHTTPMa.String(), "/http") + t.Log("libp2p http server address:", serverHTTPMa.String()) + + link, err := publisherLsys.Store( + ipld.LinkContext{Ctx: ctx}, + cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: uint64(multicodec.DagJson), + MhType: uint64(multicodec.Sha2_256), + MhLength: -1, + }, + }, + fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { + na.AssembleEntry("fish").AssignString("lobster") + na.AssembleEntry("fish1").AssignString("lobster1") + na.AssembleEntry("fish2").AssignString("lobster2") + na.AssembleEntry("fish0").AssignString("lobster0") + })) + req.NoError(err) + publisher.SetRoot(link.(cidlink.Link).Cid) + + testCases := []struct { + name string + httpPath string + expectErr bool + }{ + { + "badPath1", + "", + false, + }, + { + "badPath2", + "/", + false, + }, + { + "badPath3", + "boop/bop", + true, + }, + { + "badPath4", + "/boop/bop", + true, + }, + { + "goodPath1 no leading slash", + "boop/bop/beep", + false, + }, + { + "goodPath2 leading slash", + "/boop/bop/beep", + false, + }, + { + "goodPath3 trailing slash", + "boop/bop/beep/", + false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Plumbing to set up the test. + clientStore := &correctedMemStore{&memstore.Store{ + Bag: make(map[string][]byte), + }} + clientLsys := cidlink.DefaultLinkSystem() + clientLsys.TrustedStorage = true + clientLsys.SetReadStorage(clientStore) + clientLsys.SetWriteStorage(clientStore) + clientSync := ipnisync.NewSync(clientLsys, nil) + + httpPath := strings.TrimPrefix(tc.httpPath, "/") + var maddr multiaddr.Multiaddr + if httpPath != "" { + httpath, err := multiaddr.NewComponent("httpath", url.PathEscape(httpPath)) + req.NoError(err) + maddr = multiaddr.Join(serverHTTPMa, httpath) + } else { + maddr = serverHTTPMa + } + pubInfo := peer.AddrInfo{ + ID: publisher.ID(), + Addrs: []multiaddr.Multiaddr{maddr}, + } + t.Log("Syncing to publisher at:", pubInfo.Addrs) + clientSyncer, err := clientSync.NewSyncer(pubInfo) + req.NoError(err) + + headCid, err := clientSyncer.GetHead(ctx) + + if tc.expectErr { + req.Error(err) + return + } + + req.NoError(err) + req.Equal(link.(cidlink.Link).Cid, headCid) + + clientSyncer.Sync(ctx, headCid, selectorparse.CommonSelector_MatchPoint) + require.NoError(t, err) + + // Assert that data is loadable from the link system. + wantLink := cidlink.Link{Cid: headCid} + node, err := clientLsys.Load(ipld.LinkContext{Ctx: ctx}, wantLink, basicnode.Prototype.Any) + require.NoError(t, err) + + // Assert synced node link matches the computed link, i.e. is spec-compliant. + gotLink, err := clientLsys.ComputeLink(wantLink.Prototype(), node) + require.NoError(t, err) + require.Equal(t, gotLink, wantLink, "computed %s but got %s", gotLink.String(), wantLink.String()) + }) + } +} + func mapKeys(t *testing.T, n ipld.Node) []string { var keys []string require.Equal(t, n.Kind(), datamodel.Kind_Map) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index 5950e36..f310e8f 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -23,7 +23,6 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" libp2phttp "github.com/libp2p/go-libp2p/p2p/http" - "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multihash" ) @@ -38,11 +37,13 @@ var errHeadFromUnexpectedPeer = errors.New("found head signed from an unexpected // Sync provides sync functionality for use with all http syncs. type Sync struct { blockHook func(peer.ID, cid.Cid) - client *http.Client + client http.Client lsys ipld.LinkSystem + timeout time.Duration // libp2phttp clientHost *libp2phttp.HTTPHost + authPeerID bool } // Syncer provides sync functionality for a single sync with a peer. @@ -55,57 +56,42 @@ type Syncer struct { } // NewSync creates a new Sync. -func NewSync(lsys ipld.LinkSystem, client *http.Client, blockHook func(peer.ID, cid.Cid)) *Sync { - if client == nil { - client = &http.Client{ - Timeout: defaultHttpTimeout, - } - } - return &Sync{ - blockHook: blockHook, - client: client, - lsys: lsys, - } -} +func NewSync(lsys ipld.LinkSystem, blockHook func(peer.ID, cid.Cid), options ...ClientOption) *Sync { + opts := getClientOpts(options) -func NewLibp2pSync(lsys ipld.LinkSystem, clientHost *libp2phttp.HTTPHost, blockHook func(peer.ID, cid.Cid)) *Sync { return &Sync{ blockHook: blockHook, - lsys: lsys, - - clientHost: clientHost, + client: http.Client{ + Timeout: opts.timeout, + }, + clientHost: &libp2phttp.HTTPHost{ + StreamHost: opts.streamHost, + }, + lsys: lsys, + authPeerID: opts.authPeerID, + timeout: opts.timeout, } } // NewSyncer creates a new Syncer to use for a single sync operation against a -// peer. A value for peerID is optional for the HTTP transport. -func (s *Sync) NewSyncer(peerID peer.ID, peerAddrs []multiaddr.Multiaddr) (*Syncer, error) { - peerInfo := peer.AddrInfo{ - ID: peerID, - Addrs: peerAddrs, +// peer. A value for peerInfo.ID is optional for the HTTP transport. +func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { + var cli http.Client + var httpClient *http.Client + var err error + if s.authPeerID { + cli, err = s.clientHost.NamespacedClient(ProtocolID, peerInfo, libp2phttp.ServerMustAuthenticatePeerID) + } else { + cli, err = s.clientHost.NamespacedClient(ProtocolID, peerInfo) } - if s.clientHost != nil { - return s.newLibp2pSyncer(peerInfo) - } - return s.newSyncer(peerInfo) -} - -func (s *Sync) newLibp2pSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { - httpClient, err := s.clientHost.NamespacedClient(ProtocolID, peerInfo) if err != nil { - return nil, err + log.Warnw("Cannot create libp2phttp client. Server is not a libp2phttp server. Using plain http", "err", err) + httpClient = &s.client + } else { + httpClient = &cli } + httpClient.Timeout = s.timeout - return &Syncer{ - client: &httpClient, - peerID: peerInfo.ID, - rootURL: url.URL{Path: IpniPath}, - urls: nil, - sync: s, - }, nil -} - -func (s *Sync) newSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { urls := make([]*url.URL, len(peerInfo.Addrs)) for i, addr := range peerInfo.Addrs { u, err := maurl.ToURL(addr) @@ -116,7 +102,7 @@ func (s *Sync) newSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { } return &Syncer{ - client: s.client, + client: httpClient, peerID: peerInfo.ID, rootURL: *urls[0], urls: urls[1:], @@ -126,6 +112,9 @@ func (s *Sync) newSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { func (s *Sync) Close() { s.client.CloseIdleConnections() + if s.clientHost != nil { + s.clientHost.Close() + } } // GetHead fetches the head of the peer's advertisement chain. @@ -150,10 +139,12 @@ func (s *Syncer) GetHead(ctx context.Context) (cid.Cid, error) { return cid.Undef, errHeadFromUnexpectedPeer } - // TODO: Do something with signedHead.Topic. - // - // Should it be returned (and for what purpose)? - // Is it needed to construct the advertisement fetch URL? + // TODO: Check that the returned topic, if any, matches the expected topic. + //if signedHead.Topic != nil && *signedHead.Topic != "" && expectedTopic != "" { + // if *signedHead.Topic != expectedTopic { + // return nil, ErrTopicMismatch + // } + //} return signedHead.Head.(cidlink.Link).Cid, nil } diff --git a/dagsync/ipnisync/sync_test.go b/dagsync/ipnisync/sync_test.go index 976afb4..c748b27 100644 --- a/dagsync/ipnisync/sync_test.go +++ b/dagsync/ipnisync/sync_test.go @@ -86,8 +86,12 @@ func TestIPNISync_NFTStorage_DigestCheck(t *testing.T) { pubmaddr, err := maurl.FromURL(puburl) require.NoError(t, err) - sync := ipnisync.NewSync(ls, http.DefaultClient, nil) - syncer, err := sync.NewSyncer(pubid, []multiaddr.Multiaddr{pubmaddr}) + sync := ipnisync.NewSync(ls, nil) + pubInfo := peer.AddrInfo{ + ID: pubid, + Addrs: []multiaddr.Multiaddr{pubmaddr}, + } + syncer, err := sync.NewSyncer(pubInfo) require.NoError(t, err) head, err := syncer.GetHead(ctx) @@ -133,7 +137,7 @@ func TestIPNIsync_AcceptsSpecCompliantDagJson(t *testing.T) { publs.SetWriteStorage(pubstore) publs.SetReadStorage(pubstore) - pub, err := ipnisync.NewPublisher("0.0.0.0:0", publs, pubPrK, ipnisync.WithHeadTopic(testTopic), ipnisync.WithServer(true)) + pub, err := ipnisync.NewPublisher(publs, pubPrK, ipnisync.WithHeadTopic(testTopic), ipnisync.WithHTTPListenAddrs("0.0.0.0:0")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, pub.Close()) }) @@ -161,8 +165,12 @@ func TestIPNIsync_AcceptsSpecCompliantDagJson(t *testing.T) { ls.SetWriteStorage(store) ls.SetReadStorage(store) - sync := ipnisync.NewSync(ls, http.DefaultClient, nil) - syncer, err := sync.NewSyncer(pubID, pub.Addrs()) + sync := ipnisync.NewSync(ls, nil) + pubInfo := peer.AddrInfo{ + ID: pubID, + Addrs: pub.Addrs(), + } + syncer, err := sync.NewSyncer(pubInfo) require.NoError(t, err) head, err := syncer.GetHead(ctx) @@ -197,7 +205,7 @@ func TestIPNIsync_NotFoundReturnsContentNotFoundErr(t *testing.T) { return nil, ipld.ErrNotExists{} } - pub, err := ipnisync.NewPublisher("0.0.0.0:0", publs, pubPrK, ipnisync.WithServer(true)) + pub, err := ipnisync.NewPublisher(publs, pubPrK, ipnisync.WithHTTPListenAddrs("0.0.0.0:0")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, pub.Close()) }) @@ -206,8 +214,12 @@ func TestIPNIsync_NotFoundReturnsContentNotFoundErr(t *testing.T) { ls.SetWriteStorage(store) ls.SetReadStorage(store) - sync := ipnisync.NewSync(ls, http.DefaultClient, nil) - syncer, err := sync.NewSyncer(pubID, pub.Addrs()) + sync := ipnisync.NewSync(ls, nil) + pubInfo := peer.AddrInfo{ + ID: pubID, + Addrs: pub.Addrs(), + } + syncer, err := sync.NewSyncer(pubInfo) require.NoError(t, err) mh, err := multihash.Sum([]byte("fish"), multihash.SHA2_256, -1) diff --git a/dagsync/subscriber.go b/dagsync/subscriber.go index 28b1f6d..3993556 100644 --- a/dagsync/subscriber.go +++ b/dagsync/subscriber.go @@ -196,6 +196,12 @@ func NewSubscriber(host host.Host, ds datastore.Batching, lsys ipld.LinkSystem, ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) all := ssb.ExploreAll(ssb.ExploreRecursiveEdge()) + var httpTimeout time.Duration + if opts.httpClient != nil { + httpTimeout = opts.httpClient.Timeout + } + ipniSync := ipnisync.NewSync(lsys, blockHook, ipnisync.ClientStreamHost(host), ipnisync.ClientTimeout(httpTimeout)) + s := &Subscriber{ host: host, @@ -209,7 +215,7 @@ func NewSubscriber(host host.Host, ds datastore.Batching, lsys ipld.LinkSystem, rmEventChan: make(chan chan<- SyncFinished), dtSync: dtSync, - ipniSync: ipnisync.NewSync(lsys, opts.httpClient, blockHook), + ipniSync: ipniSync, httpPeerstore: httpPeerstore, @@ -766,7 +772,17 @@ func (s *Subscriber) makeSyncer(peerInfo peer.AddrInfo, doUpdate bool) (Syncer, // Store this http address so that future calls to sync will work without a // peerAddr (given that it happens within the TTL) s.httpPeerstore.AddAddrs(peerInfo.ID, httpAddrs, tempAddrTTL) - syncer, err := s.ipniSync.NewSyncer(peerInfo.ID, httpAddrs) + + //p2pSyncer, err := s.p2pIpniSync.NewSyncer(peerInfo.ID, peerInfo.Addrs) + //if err != nil { + // return nil, nil, fmt.Errorf("cannot create p2p ipni-sync handler: %w", err) + //} + + httpPeerInfo := peer.AddrInfo{ + ID: peerInfo.ID, + Addrs: httpAddrs, + } + syncer, err := s.ipniSync.NewSyncer(httpPeerInfo) if err != nil { return nil, nil, fmt.Errorf("cannot create ipni-sync handler: %w", err) } @@ -794,6 +810,7 @@ func (s *Subscriber) makeSyncer(peerInfo peer.AddrInfo, doUpdate bool) (Syncer, update = func() {} } } + // Not an httpPeerAddr, so use the dtSync. return s.dtSync.NewSyncer(peerInfo.ID, s.topicName), update, nil } diff --git a/dagsync/subscriber_test.go b/dagsync/subscriber_test.go index c2f4cc3..9e39caa 100644 --- a/dagsync/subscriber_test.go +++ b/dagsync/subscriber_test.go @@ -884,7 +884,7 @@ func (b dagsyncPubSubBuilder) Build(t *testing.T, topicName string, pubSys hostS var pub dagsync.Publisher var err error if b.IsHttp { - pub, err = ipnisync.NewPublisher("127.0.0.1:0", pubSys.lsys, pubSys.privKey, ipnisync.WithHeadTopic(topicName)) + pub, err = ipnisync.NewPublisher(pubSys.lsys, pubSys.privKey, ipnisync.WithHeadTopic(topicName), ipnisync.WithHTTPListenAddrs("127.0.0.1:0")) require.NoError(t, err) require.NoError(t, test.WaitForHttpPublisher(pub)) } else { From a8fe746f0bc902d4af6e37928923ff5b4ad815ee Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 18 Aug 2023 11:09:27 -0700 Subject: [PATCH 03/29] Subscriber does not use existing http client - Update comments - Add HttpTimeout option for subscriber --- dagsync/ipnisync/option.go | 27 +++++++++++++++++++++------ dagsync/ipnisync/publisher_test.go | 2 +- dagsync/ipnisync/sync.go | 24 ++++++++++-------------- dagsync/option.go | 24 ++++++++++++++---------- dagsync/subscriber.go | 14 +++----------- 5 files changed, 49 insertions(+), 42 deletions(-) diff --git a/dagsync/ipnisync/option.go b/dagsync/ipnisync/option.go index ae10945..ab8a46b 100644 --- a/dagsync/ipnisync/option.go +++ b/dagsync/ipnisync/option.go @@ -34,6 +34,13 @@ func getOpts(opts []Option) (config, error) { return cfg, nil } +// WithHTTPListenAddrs sets the HTTP addresses to listen on. These are in +// addresses:port format and may be prefixed with "https://" or "http://" or to +// specify whether or not TLS is required. If there is no prefix, then one is +// assumed based on the value specified by WithRequireTLS. +// +// Setting HTTP listen addresses is optional when a stream host is provided by +// the WithStreamHost option. func WithHTTPListenAddrs(addrs ...string) Option { return func(c *config) error { c.httpAddrs = append(c.httpAddrs, addrs...) @@ -79,6 +86,8 @@ func WithRequireTLS(require bool) Option { } } +// WithStreamHost specifies an optional stream based libp2p host used to do +// HTTP over libp2p streams. func WithStreamHost(h host.Host) Option { return func(c *config) error { c.streamHost = h @@ -87,9 +96,9 @@ func WithStreamHost(h host.Host) Option { } type clientConfig struct { - authPeerID bool - timeout time.Duration - streamHost host.Host + authPeerID bool + httpTimeout time.Duration + streamHost host.Host } // Option is a function that sets a value in a config. @@ -104,18 +113,24 @@ func getClientOpts(opts []ClientOption) clientConfig { return cfg } -func ClientAuthPeerID(require bool) ClientOption { +// ClientAuthServerPeerID tells the sync client that we MUST +// authenticate the Server's PeerID. +func ClientAuthServerPeerID(require bool) ClientOption { return func(c *clientConfig) { c.authPeerID = require } } -func ClientTimeout(to time.Duration) ClientOption { +// ClientHTTPTimeout specifies a time limit for HTTP requests made by the sync +// client. A value of zero means no timeout. +func ClientHTTPTimeout(to time.Duration) ClientOption { return func(c *clientConfig) { - c.timeout = to + c.httpTimeout = to } } +// ClientStreamHost specifies an optional stream based libp2p host used by the +// sync client to do HTTP over libp2p streams. func ClientStreamHost(h host.Host) ClientOption { return func(c *clientConfig) { c.streamHost = h diff --git a/dagsync/ipnisync/publisher_test.go b/dagsync/ipnisync/publisher_test.go index 83dc6f9..5a71263 100644 --- a/dagsync/ipnisync/publisher_test.go +++ b/dagsync/ipnisync/publisher_test.go @@ -420,7 +420,7 @@ func TestHandlerPath(t *testing.T) { maddr = serverHTTPMa } pubInfo := peer.AddrInfo{ - ID: publisher.ID(), + //ID: publisher.ID(), // optional Addrs: []multiaddr.Multiaddr{maddr}, } t.Log("Syncing to publisher at:", pubInfo.Addrs) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index f310e8f..6b1299c 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -28,18 +28,14 @@ import ( const ProtocolID = protocol.ID("/ipnisync/v1") -const defaultHttpTimeout = 10 * time.Second - var log = logging.Logger("dagsync/ipnisync") -var errHeadFromUnexpectedPeer = errors.New("found head signed from an unexpected peer") - // Sync provides sync functionality for use with all http syncs. type Sync struct { - blockHook func(peer.ID, cid.Cid) - client http.Client - lsys ipld.LinkSystem - timeout time.Duration + blockHook func(peer.ID, cid.Cid) + client http.Client + lsys ipld.LinkSystem + httpTimeout time.Duration // libp2phttp clientHost *libp2phttp.HTTPHost @@ -62,14 +58,14 @@ func NewSync(lsys ipld.LinkSystem, blockHook func(peer.ID, cid.Cid), options ... return &Sync{ blockHook: blockHook, client: http.Client{ - Timeout: opts.timeout, + Timeout: opts.httpTimeout, }, clientHost: &libp2phttp.HTTPHost{ StreamHost: opts.streamHost, }, - lsys: lsys, - authPeerID: opts.authPeerID, - timeout: opts.timeout, + lsys: lsys, + authPeerID: opts.authPeerID, + httpTimeout: opts.httpTimeout, } } @@ -90,7 +86,7 @@ func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { } else { httpClient = &cli } - httpClient.Timeout = s.timeout + httpClient.Timeout = s.httpTimeout urls := make([]*url.URL, len(peerInfo.Addrs)) for i, addr := range peerInfo.Addrs { @@ -136,7 +132,7 @@ func (s *Syncer) GetHead(ctx context.Context) (cid.Cid, error) { if s.peerID == "" { log.Warn("cannot verify publisher signature without peer ID") } else if signerID != s.peerID { - return cid.Undef, errHeadFromUnexpectedPeer + return cid.Undef, errors.New("found head signed by an unexpected peer") } // TODO: Check that the returned topic, if any, matches the expected topic. diff --git a/dagsync/option.go b/dagsync/option.go index eae5d2a..2300c6e 100644 --- a/dagsync/option.go +++ b/dagsync/option.go @@ -2,7 +2,6 @@ package dagsync import ( "fmt" - "net/http" "time" "github.com/ipfs/go-cid" @@ -24,6 +23,8 @@ const ( // Maximum number of in-prgress graphsync requests. defaultGsMaxInRequests = 1024 defaultGsMaxOutRequests = 1024 + // defaultHttpTimeout is time limit for requests made by the HTTP client. + defaultHttpTimeout = 10 * time.Second ) type LastKnownSyncFunc func(peer.ID) (cid.Cid, bool) @@ -34,8 +35,8 @@ type config struct { topic *pubsub.Topic - blockHook BlockHookFunc - httpClient *http.Client + blockHook BlockHookFunc + httpTimeout time.Duration idleHandlerTTL time.Duration lastKnownSync LastKnownSyncFunc @@ -61,6 +62,7 @@ type Option func(*config) error func getOpts(opts []Option) (config, error) { cfg := config{ addrTTL: defaultAddrTTL, + httpTimeout: defaultHttpTimeout, idleHandlerTTL: defaultIdleHandlerTTL, segDepthLimit: defaultSegDepthLimit, gsMaxInRequests: defaultGsMaxInRequests, @@ -93,18 +95,20 @@ func Topic(topic *pubsub.Topic) Option { } } -// HttpClient provides Subscriber with an existing http client. -func HttpClient(client *http.Client) Option { +// HttpTimeout specifies a time limit for HTTP requests made by the sync +// HTTP client. A value of zero means no timeout. +func HttpTimeout(to time.Duration) Option { return func(c *config) error { - c.httpClient = client + c.httpTimeout = to return nil } } -// BlockHook adds a hook that is run when a block is received via Subscriber.Sync along with a -// SegmentSyncActions to control the sync flow if segmented sync is enabled. -// Note that if segmented sync is disabled, calls on SegmentSyncActions will have no effect. -// See: SegmentSyncActions, SegmentDepthLimit, ScopedBlockHook. +// BlockHook adds a hook that is run when a block is received via +// Subscriber.Sync along with a SegmentSyncActions to control the sync flow if +// segmented sync is enabled. Note that if segmented sync is disabled, calls on +// SegmentSyncActions will have no effect. See: SegmentSyncActions, +// SegmentDepthLimit, ScopedBlockHook. func BlockHook(blockHook BlockHookFunc) Option { return func(c *config) error { c.blockHook = blockHook diff --git a/dagsync/subscriber.go b/dagsync/subscriber.go index 3993556..1c8c001 100644 --- a/dagsync/subscriber.go +++ b/dagsync/subscriber.go @@ -196,11 +196,9 @@ func NewSubscriber(host host.Host, ds datastore.Batching, lsys ipld.LinkSystem, ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) all := ssb.ExploreAll(ssb.ExploreRecursiveEdge()) - var httpTimeout time.Duration - if opts.httpClient != nil { - httpTimeout = opts.httpClient.Timeout - } - ipniSync := ipnisync.NewSync(lsys, blockHook, ipnisync.ClientStreamHost(host), ipnisync.ClientTimeout(httpTimeout)) + ipniSync := ipnisync.NewSync(lsys, blockHook, + ipnisync.ClientStreamHost(host), + ipnisync.ClientHTTPTimeout(opts.httpTimeout)) s := &Subscriber{ host: host, @@ -772,12 +770,6 @@ func (s *Subscriber) makeSyncer(peerInfo peer.AddrInfo, doUpdate bool) (Syncer, // Store this http address so that future calls to sync will work without a // peerAddr (given that it happens within the TTL) s.httpPeerstore.AddAddrs(peerInfo.ID, httpAddrs, tempAddrTTL) - - //p2pSyncer, err := s.p2pIpniSync.NewSyncer(peerInfo.ID, peerInfo.Addrs) - //if err != nil { - // return nil, nil, fmt.Errorf("cannot create p2p ipni-sync handler: %w", err) - //} - httpPeerInfo := peer.AddrInfo{ ID: peerInfo.ID, Addrs: httpAddrs, From 37ef775b2ec64f70e20ca8cb4d02713ba249e5be Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 18 Aug 2023 13:14:40 -0700 Subject: [PATCH 04/29] Fix handler path processing Add option to use retryable http client --- dagsync/ipnisync/ipnipath.go | 2 +- dagsync/ipnisync/option.go | 20 ++++++-- dagsync/ipnisync/publisher.go | 80 +++++++++++++----------------- dagsync/ipnisync/publisher_test.go | 6 +-- dagsync/ipnisync/sync.go | 32 +++++++++++- dagsync/option.go | 19 ++++++- dagsync/subscriber.go | 3 +- dagsync/test/util.go | 2 +- go.mod | 2 + go.sum | 6 +++ 10 files changed, 113 insertions(+), 59 deletions(-) diff --git a/dagsync/ipnisync/ipnipath.go b/dagsync/ipnisync/ipnipath.go index ac2e7d4..41077ca 100644 --- a/dagsync/ipnisync/ipnipath.go +++ b/dagsync/ipnisync/ipnipath.go @@ -4,4 +4,4 @@ import "path" const protoVersion = "v1" -var IpniPath = path.Join("/ipni", protoVersion, "ad") +var IPNIPath = path.Join("/ipni", protoVersion, "ad") diff --git a/dagsync/ipnisync/option.go b/dagsync/ipnisync/option.go index ab8a46b..77662fc 100644 --- a/dagsync/ipnisync/option.go +++ b/dagsync/ipnisync/option.go @@ -96,9 +96,13 @@ func WithStreamHost(h host.Host) Option { } type clientConfig struct { - authPeerID bool - httpTimeout time.Duration - streamHost host.Host + authPeerID bool + streamHost host.Host + + httpTimeout time.Duration + httpRetryMax int + httpRetryWaitMin time.Duration + httpRetryWaitMax time.Duration } // Option is a function that sets a value in a config. @@ -136,3 +140,13 @@ func ClientStreamHost(h host.Host) ClientOption { c.streamHost = h } } + +// ClientHTTPRetry configures a retriable HTTP client. Setting retryMax to +// zero, the default, disables the retriable client. +func ClientHTTPRetry(retryMax int, waitMin, waitMax time.Duration) ClientOption { + return func(c *clientConfig) { + c.httpRetryMax = retryMax + c.httpRetryWaitMin = waitMin + c.httpRetryWaitMax = waitMax + } +} diff --git a/dagsync/ipnisync/publisher.go b/dagsync/ipnisync/publisher.go index b60ef48..c6e4e99 100644 --- a/dagsync/ipnisync/publisher.go +++ b/dagsync/ipnisync/publisher.go @@ -60,14 +60,22 @@ func NewPublisher(lsys ipld.LinkSystem, privKey ic.PrivKey, options ...Option) ( } pub := &Publisher{ - lsys: lsys, - handlerPath: strings.TrimPrefix(IpniPath, "/"), - peerID: peerID, - privKey: privKey, - topic: opts.topic, + lsys: lsys, + peerID: peerID, + privKey: privKey, + topic: opts.topic, } + // Construct expected request path prefix. If server is started this will + // get stripped off. If using an external server, look for this path when + // handling requests. + var handlerPath string opts.handlerPath = strings.TrimPrefix(opts.handlerPath, "/") + if opts.handlerPath != "" { + handlerPath = path.Join(opts.handlerPath, IPNIPath) + } else { + handlerPath = strings.TrimPrefix(IPNIPath, "/") + } if !opts.startServer { httpListenAddrs, err := httpAddrsToMultiaddrs(opts.httpAddrs, opts.requireTLS, opts.handlerPath) @@ -80,9 +88,7 @@ func NewPublisher(lsys ipld.LinkSystem, privKey ic.PrivKey, options ...Option) ( // If the server is not started, the handlerPath does not get stripped // from the HTTP request, so leave it as part of the prefix to match in // the SetveHTTP handler. - if opts.handlerPath != "" { - pub.handlerPath = path.Join(opts.handlerPath, pub.handlerPath) - } + pub.handlerPath = handlerPath pub.httpAddrs = httpListenAddrs return pub, nil } @@ -104,23 +110,17 @@ func NewPublisher(lsys ipld.LinkSystem, privKey ic.PrivKey, options ...Option) ( } pub.pubHost = publisherHost - if opts.handlerPath == "" { - opts.handlerPath = "/" - } else { - if !strings.HasPrefix(opts.handlerPath, "/") { - opts.handlerPath = "/" + opts.handlerPath - } - } - // Here is where this Publisher is attached as a request handler. This - // mounts the "/ipnisync/v1" protocol at "/ipni/v1/ad/". If - // opts.handlerPath is "/foo/", this mounts it at "/foo/ipni/v1/ad/". - // libp2phttp manages this mapping and clients can learn about the mapping - // at .well-known/libp2p. + // mounts the "/ipnisync/v1" protocol at "/opt_handler_path/ipni/v1/ad/", + // where opt_handler_path is the optional user specified handler path. If + // opts.handlerPath is "/foo/", this mounts it at "/foo/ipni/v1/ad/". This + // Publisher will only receive requests whose path begins with the + // handlerPath. libp2phttp manages this mapping and clients can learn about + // the mapping at .well-known/libp2p. // // In this case we also want the HTTP handler to not even know about the // prefix, so we use the stdlib http.StripPrefix. - publisherHost.SetHTTPHandlerAtPath(ProtocolID, opts.handlerPath, pub) + publisherHost.SetHTTPHandlerAtPath(ProtocolID, "/"+handlerPath, pub) go publisherHost.Serve() @@ -151,26 +151,6 @@ func NewPublisherWithoutServer(address, handlerPath string, lsys ipld.LinkSystem return NewPublisher(lsys, privKey, WithHTTPListenAddrs(address), WithHandlerPath(handlerPath), WithStartServer(false)) } -// NewPublisherHandler returns a Publisher for use as an http.Handler. Does not -// listen or know about a url prefix. -func NewPublisherHandler(lsys ipld.LinkSystem, privKey ic.PrivKey) (*Publisher, error) { - if privKey == nil { - return nil, errors.New("private key required to sign head requests") - } - peerID, err := peer.IDFromPrivateKey(privKey) - if err != nil { - return nil, fmt.Errorf("could not get peer id from private key: %w", err) - } - - return &Publisher{ - addr: nil, - lsys: lsys, - handlerPath: strings.TrimPrefix(IpniPath, "/"), - peerID: peerID, - privKey: privKey, - }, nil -} - // Addrs returns the addresses, as []multiaddress, that the Publisher is // listening on. // @@ -212,14 +192,22 @@ func (p *Publisher) Close() error { // ServeHTTP implements the http.Handler interface. func (p *Publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // A URL path from http will have a leading "/". A URL from libp2phttp will not. - urlPath := strings.TrimPrefix(r.URL.Path, "/") - if p.handlerPath != "" && !strings.HasPrefix(urlPath, p.handlerPath) { - http.Error(w, "invalid request path: "+urlPath, http.StatusBadRequest) + // If we expect publisher requests to have a prefix in the request path, + // then check for the expected prefix.. This happens when using an external + // server with this Publisher as the request handler. + if p.handlerPath != "" { + // A URL path from http will have a leading "/". A URL from libp2phttp will not. + urlPath := strings.TrimPrefix(r.URL.Path, "/") + if !strings.HasPrefix(urlPath, p.handlerPath) { + http.Error(w, "invalid request path: "+r.URL.Path, http.StatusBadRequest) + return + } + } else if path.Dir(r.URL.Path) != "." { + http.Error(w, "invalid request path: "+r.URL.Path, http.StatusBadRequest) return } - ask := path.Base(urlPath) + ask := path.Base(r.URL.Path) if ask == "head" { // serve the head p.lock.Lock() diff --git a/dagsync/ipnisync/publisher_test.go b/dagsync/ipnisync/publisher_test.go index 5a71263..d77e355 100644 --- a/dagsync/ipnisync/publisher_test.go +++ b/dagsync/ipnisync/publisher_test.go @@ -286,7 +286,7 @@ func TestNewPublisherForListener(t *testing.T) { resp := &mockResponseWriter{} u := &url.URL{ - Path: path.Join("/", handlerPath, ipnisync.IpniPath, "/head"), + Path: path.Join("/", handlerPath, ipnisync.IPNIPath, "/head"), } subject.ServeHTTP(resp, &http.Request{URL: u}) @@ -364,12 +364,12 @@ func TestHandlerPath(t *testing.T) { { "badPath1", "", - false, + true, }, { "badPath2", "/", - false, + true, }, { "badPath3", diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index 6b1299c..50244ff 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -10,6 +10,7 @@ import ( "net/url" "time" + "github.com/hashicorp/go-retryablehttp" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "github.com/ipld/go-ipld-prime" @@ -40,6 +41,7 @@ type Sync struct { // libp2phttp clientHost *libp2phttp.HTTPHost authPeerID bool + rclient *retryablehttp.Client } // Syncer provides sync functionality for a single sync with a peer. @@ -55,7 +57,7 @@ type Syncer struct { func NewSync(lsys ipld.LinkSystem, blockHook func(peer.ID, cid.Cid), options ...ClientOption) *Sync { opts := getClientOpts(options) - return &Sync{ + s := &Sync{ blockHook: blockHook, client: http.Client{ Timeout: opts.httpTimeout, @@ -67,6 +69,19 @@ func NewSync(lsys ipld.LinkSystem, blockHook func(peer.ID, cid.Cid), options ... authPeerID: opts.authPeerID, httpTimeout: opts.httpTimeout, } + + if opts.httpRetryMax != 0 { + // Instantiate retryable HTTP client used by dagsync/ipnisync. + s.rclient = &retryablehttp.Client{ + RetryWaitMin: opts.httpRetryWaitMin, + RetryWaitMax: opts.httpRetryWaitMax, + RetryMax: opts.httpRetryMax, + CheckRetry: retryablehttp.DefaultRetryPolicy, + Backoff: retryablehttp.DefaultBackoff, + } + } + + return s } // NewSyncer creates a new Syncer to use for a single sync operation against a @@ -88,13 +103,26 @@ func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { } httpClient.Timeout = s.httpTimeout + if s.rclient != nil { + // Instantiate retryable HTTP client used by dagsync/ipnisync. + rclient := &retryablehttp.Client{ + HTTPClient: httpClient, + RetryWaitMin: s.rclient.RetryWaitMin, + RetryWaitMax: s.rclient.RetryWaitMax, + RetryMax: s.rclient.RetryMax, + CheckRetry: retryablehttp.DefaultRetryPolicy, + Backoff: retryablehttp.DefaultBackoff, + } + httpClient = rclient.StandardClient() + } + urls := make([]*url.URL, len(peerInfo.Addrs)) for i, addr := range peerInfo.Addrs { u, err := maurl.ToURL(addr) if err != nil { return nil, err } - urls[i] = u.JoinPath(IpniPath) + urls[i] = u.JoinPath(IPNIPath) } return &Syncer{ diff --git a/dagsync/option.go b/dagsync/option.go index 2300c6e..7d9e391 100644 --- a/dagsync/option.go +++ b/dagsync/option.go @@ -35,8 +35,7 @@ type config struct { topic *pubsub.Topic - blockHook BlockHookFunc - httpTimeout time.Duration + blockHook BlockHookFunc idleHandlerTTL time.Duration lastKnownSync LastKnownSyncFunc @@ -53,6 +52,11 @@ type config struct { gsMaxOutRequests uint64 strictAdsSelSeq bool + + httpTimeout time.Duration + httpRetryMax int + httpRetryWaitMin time.Duration + httpRetryWaitMax time.Duration } // Option is a function that sets a value in a config. @@ -104,6 +108,17 @@ func HttpTimeout(to time.Duration) Option { } } +// RetryableHTTPClient configures a retriable HTTP client. Setting retryMax to +// zero, the default, disables the retriable client. +func ClientHTTPRetry(retryMax int, waitMin, waitMax time.Duration) Option { + return func(c *config) error { + c.httpRetryMax = retryMax + c.httpRetryWaitMin = waitMin + c.httpRetryWaitMax = waitMax + return nil + } +} + // BlockHook adds a hook that is run when a block is received via // Subscriber.Sync along with a SegmentSyncActions to control the sync flow if // segmented sync is enabled. Note that if segmented sync is disabled, calls on diff --git a/dagsync/subscriber.go b/dagsync/subscriber.go index 1c8c001..f2f47a8 100644 --- a/dagsync/subscriber.go +++ b/dagsync/subscriber.go @@ -198,7 +198,8 @@ func NewSubscriber(host host.Host, ds datastore.Batching, lsys ipld.LinkSystem, ipniSync := ipnisync.NewSync(lsys, blockHook, ipnisync.ClientStreamHost(host), - ipnisync.ClientHTTPTimeout(opts.httpTimeout)) + ipnisync.ClientHTTPTimeout(opts.httpTimeout), + ipnisync.ClientHTTPRetry(opts.httpRetryMax, opts.httpRetryWaitMin, opts.httpRetryWaitMax)) s := &Subscriber{ host: host, diff --git a/dagsync/test/util.go b/dagsync/test/util.go index da66c96..f9d625c 100644 --- a/dagsync/test/util.go +++ b/dagsync/test/util.go @@ -319,7 +319,7 @@ func WaitForHttpPublisher(publisher TestPublisher) error { if err != nil { return err } - headURL.Path = path.Join(headURL.Path, ipnisync.IpniPath, "head") + headURL.Path = path.Join(headURL.Path, ipnisync.IPNIPath, "head") ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() diff --git a/go.mod b/go.mod index a02375a..fcf010c 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7 github.com/gammazero/channelqueue v0.2.1 github.com/hashicorp/go-multierror v1.1.1 + github.com/hashicorp/go-retryablehttp v0.7.4 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-graphsync v0.14.7 @@ -63,6 +64,7 @@ require ( github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c // indirect github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/huin/goupnp v1.2.0 // indirect github.com/ipfs/go-block-format v0.0.3 // indirect diff --git a/go.sum b/go.sum index 4abe12a..3f9f9d1 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,14 @@ github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e/go.mod h1:I github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= +github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= From 4c6762ad7824688a959d79c8eab557c6be5b7d5d Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 18 Aug 2023 18:22:26 -0700 Subject: [PATCH 05/29] Change option name --- dagsync/option.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dagsync/option.go b/dagsync/option.go index 7d9e391..da7b421 100644 --- a/dagsync/option.go +++ b/dagsync/option.go @@ -1,6 +1,7 @@ package dagsync import ( + "errors" "fmt" "time" @@ -110,8 +111,14 @@ func HttpTimeout(to time.Duration) Option { // RetryableHTTPClient configures a retriable HTTP client. Setting retryMax to // zero, the default, disables the retriable client. -func ClientHTTPRetry(retryMax int, waitMin, waitMax time.Duration) Option { +func RetryableHTTPClient(retryMax int, waitMin, waitMax time.Duration) Option { return func(c *config) error { + if waitMin > waitMax { + return errors.New("minimum retry wait time cannot be greater than maximum") + } + if retryMax < 0 { + retryMax = 0 + } c.httpRetryMax = retryMax c.httpRetryWaitMin = waitMin c.httpRetryWaitMax = waitMax From f9322b04b0d54bfa9eb81980308e0acd398441c1 Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 18 Aug 2023 20:19:02 -0700 Subject: [PATCH 06/29] Update comments --- dagsync/ipnisync/option.go | 6 ++++-- dagsync/ipnisync/publisher.go | 36 +++++++++++++---------------------- dagsync/ipnisync/sync.go | 34 ++++++++++++++------------------- dagsync/option.go | 6 +++--- 4 files changed, 34 insertions(+), 48 deletions(-) diff --git a/dagsync/ipnisync/option.go b/dagsync/ipnisync/option.go index 77662fc..ce17645 100644 --- a/dagsync/ipnisync/option.go +++ b/dagsync/ipnisync/option.go @@ -7,7 +7,7 @@ import ( "github.com/libp2p/go-libp2p/core/host" ) -// pubConfig contains all options for configuring Publisher. +// config contains all options for configuring Publisher. type config struct { handlerPath string startServer bool @@ -49,7 +49,9 @@ func WithHTTPListenAddrs(addrs ...string) Option { } // WithHandlerPath sets the path used to handle requests to this publisher. -// This should only include the path before the /ipni/v1/ad/ part of the path. +// This specifies the portion of the path before the implicit /ipni/v1/ad/ part +// of the path. Calling WithHandlerPath("/foo/bar") configures the publisher to +// handle HTTP requests on the path "/foo/bar/ipni/v1/ad/". func WithHandlerPath(urlPath string) Option { return func(c *config) error { c.handlerPath = urlPath diff --git a/dagsync/ipnisync/publisher.go b/dagsync/ipnisync/publisher.go index c6e4e99..88fdbbf 100644 --- a/dagsync/ipnisync/publisher.go +++ b/dagsync/ipnisync/publisher.go @@ -3,7 +3,6 @@ package ipnisync import ( "errors" "fmt" - "net" "net/http" "net/url" "path" @@ -35,7 +34,7 @@ type Publisher struct { topic string pubHost *libp2phttp.HTTPHost - // haatAddrs is returned by Addrs when not starting the server. + // httpAddrs is returned by Addrs when not starting the server. httpAddrs []multiaddr.Multiaddr } @@ -87,7 +86,7 @@ func NewPublisher(lsys ipld.LinkSystem, privKey ic.PrivKey, options ...Option) ( } // If the server is not started, the handlerPath does not get stripped // from the HTTP request, so leave it as part of the prefix to match in - // the SetveHTTP handler. + // the ServeHTTP handler. pub.handlerPath = handlerPath pub.httpAddrs = httpListenAddrs return pub, nil @@ -130,28 +129,19 @@ func NewPublisher(lsys ipld.LinkSystem, privKey ic.PrivKey, options ...Option) ( return pub, nil } -// NewPublisherForListener creates a new http publisher for an existing -// listener. When providing an existing listener, running the HTTP server -// is the caller's responsibility. ServeHTTP on the returned Publisher -// can be used to handle requests. handlerPath is the path to handle -// requests on, e.g. "ipni" for `/ipni/...` requests. -// -// DEPRECATED: use NewPublisher(lsys, privKey, WithHTTPListenAddrs(listener.Addr().String()), WithHandlerPath(handlerPath), WithStartServer(false)) -func NewPublisherForListener(listener net.Listener, handlerPath string, lsys ipld.LinkSystem, privKey ic.PrivKey) (*Publisher, error) { - return NewPublisher(lsys, privKey, WithHTTPListenAddrs(listener.Addr().String()), WithHandlerPath(handlerPath), WithStartServer(false)) -} - // NewPublisherWithoutServer creates a new http publisher for an existing -// network address. When providing an existing network address, running -// the HTTP server is the caller's responsibility. ServeHTTP on the -// returned Publisher can be used to handle requests. +// network address. When providing an existing network address, running the +// HTTP server is the caller's responsibility. ServeHTTP on the returned +// Publisher can be used to handle requests. handlerPath is the path to handle +// requests on before the /ipni/v1/ad/ portion of the path. See +// WithHandlerPath. // // DEPRECATED: use NewPublisher(lsys, privKey, WithHTTPListenAddrs(address), WithHandlerPath(handlerPath), WithStartServer(false)) func NewPublisherWithoutServer(address, handlerPath string, lsys ipld.LinkSystem, privKey ic.PrivKey, options ...Option) (*Publisher, error) { return NewPublisher(lsys, privKey, WithHTTPListenAddrs(address), WithHandlerPath(handlerPath), WithStartServer(false)) } -// Addrs returns the addresses, as []multiaddress, that the Publisher is +// Addrs returns the slice of multiaddr addresses that the Publisher is // listening on. // // If the server is not started, WithStartServer(false), then this returns the @@ -169,7 +159,7 @@ func (p *Publisher) ID() peer.ID { return p.peerID } -// Protocol returns the multihash protocol ID of the transport used by the +// Protocol returns the multiaddr protocol ID of the transport used by the // publisher. func (p *Publisher) Protocol() int { return multiaddr.P_HTTP @@ -195,21 +185,21 @@ func (p *Publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) { // If we expect publisher requests to have a prefix in the request path, // then check for the expected prefix.. This happens when using an external // server with this Publisher as the request handler. + urlPath := strings.TrimPrefix(r.URL.Path, "/") if p.handlerPath != "" { // A URL path from http will have a leading "/". A URL from libp2phttp will not. - urlPath := strings.TrimPrefix(r.URL.Path, "/") if !strings.HasPrefix(urlPath, p.handlerPath) { http.Error(w, "invalid request path: "+r.URL.Path, http.StatusBadRequest) return } - } else if path.Dir(r.URL.Path) != "." { + } else if path.Dir(urlPath) != "." { http.Error(w, "invalid request path: "+r.URL.Path, http.StatusBadRequest) return } ask := path.Base(r.URL.Path) if ask == "head" { - // serve the head + // Serve the head message. p.lock.Lock() rootCid := p.root p.lock.Unlock() @@ -229,7 +219,7 @@ func (p *Publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(marshalledMsg) return } - // interpret `ask` as a CID to serve. + // Interpret `ask` as a CID to serve. c, err := cid.Parse(ask) if err != nil { http.Error(w, "invalid request: not a cid", http.StatusBadRequest) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index 50244ff..5a91df8 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -71,13 +71,11 @@ func NewSync(lsys ipld.LinkSystem, blockHook func(peer.ID, cid.Cid), options ... } if opts.httpRetryMax != 0 { - // Instantiate retryable HTTP client used by dagsync/ipnisync. + // Configure retryable HTTP client created by calls to NewSyncer. s.rclient = &retryablehttp.Client{ RetryWaitMin: opts.httpRetryWaitMin, RetryWaitMax: opts.httpRetryWaitMax, RetryMax: opts.httpRetryMax, - CheckRetry: retryablehttp.DefaultRetryPolicy, - Backoff: retryablehttp.DefaultBackoff, } } @@ -104,7 +102,7 @@ func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { httpClient.Timeout = s.httpTimeout if s.rclient != nil { - // Instantiate retryable HTTP client used by dagsync/ipnisync. + // Instantiate retryable HTTP client. rclient := &retryablehttp.Client{ HTTPClient: httpClient, RetryWaitMin: s.rclient.RetryWaitMin, @@ -185,15 +183,14 @@ func (s *Syncer) Sync(ctx context.Context, nextCid cid.Cid, sel ipld.Node) error return fmt.Errorf("failed to traverse requested dag: %w", err) } - // We run the block hook to emulate the behavior of graphsync's - // `OnIncomingBlockHook` callback (gets called even if block is already stored - // locally). + // The blockHook callback gets called for every synced block, even if block + // is already stored locally. // // We are purposefully not doing this in the StorageReadOpener because the - // hook can do anything, including deleting the block from the block store. If - // it did that then we would not be able to continue our traversal. So instead - // we remember the blocks seen during traversal and then call the hook at the - // end when we no longer care what it does with the blocks. + // hook can do anything, including deleting the block from the block store. + // If it did that then we would not be able to continue our traversal. So + // instead we remember the blocks seen during traversal and then call the + // hook at the end when we no longer care what it does with the blocks. if s.sync.blockHook != nil { for _, c := range cids { s.sync.blockHook(s.peerID, c) @@ -204,19 +201,16 @@ func (s *Syncer) Sync(ctx context.Context, nextCid cid.Cid, sel ipld.Node) error return nil } -// walkFetch is run by a traversal of the selector. For each block that the +// walkFetch is run by a traversal of the selector. For each block that the // selector walks over, walkFetch will look to see if it can find it in the -// local data store. If it cannot, it will then go and get it over HTTP. This -// emulates way libp2p/graphsync fetches data, but the actual fetch of data is -// done over HTTP. +// local data store. If it cannot, it will then go and get it over HTTP. func (s *Syncer) walkFetch(ctx context.Context, rootCid cid.Cid, sel selector.Selector) ([]cid.Cid, error) { - // Track the order of cids we've seen during our traversal so we can call the - // block hook function in the same order. We emulate the behavior of - // graphsync's `OnIncomingBlockHook`, this means we call the blockhook even if - // we have the block locally. + // Track the order of cids seen during traversal so that the block hook + // function gets called in the same order. var traversalOrder []cid.Cid getMissingLs := cidlink.DefaultLinkSystem() - // trusted because it'll be hashed/verified on the way into the link system when fetched. + // Trusted because it will be hashed/verified on the way into the link + // system when fetched. getMissingLs.TrustedStorage = true getMissingLs.StorageReadOpener = func(lc ipld.LinkContext, l ipld.Link) (io.Reader, error) { c := l.(cidlink.Link).Cid diff --git a/dagsync/option.go b/dagsync/option.go index da7b421..3c2d49e 100644 --- a/dagsync/option.go +++ b/dagsync/option.go @@ -21,7 +21,7 @@ const ( defaultIdleHandlerTTL = time.Hour // defaultSegDepthLimit disables (-1) segmented sync by default. defaultSegDepthLimit = -1 - // Maximum number of in-prgress graphsync requests. + // Maximum number of in-progress graphsync requests. defaultGsMaxInRequests = 1024 defaultGsMaxOutRequests = 1024 // defaultHttpTimeout is time limit for requests made by the HTTP client. @@ -197,7 +197,7 @@ func RecvAnnounce(opts ...announce.Option) Option { // MaxAsyncConcurrency sets the maximum number of concurrent asynchrouous syncs // (started by announce messages). This only takes effect if there is an -// announcement reveiver configured by the RecvAnnounce option. +// announcement receiver configured by the RecvAnnounce option. func MaxAsyncConcurrency(n int) Option { return func(c *config) error { if n != 0 { @@ -267,7 +267,7 @@ func WithStopAdCid(stopAd cid.Cid) SyncOption { } } -// WithResyncAds causes the current sync to ignore anvertisements that have been +// WithResyncAds causes the current sync to ignore advertisements that have been // previously synced. When true, sync does not record the latest synced CID or // send sync finished notification. func WithAdsResync(resync bool) SyncOption { From b91b8e8b2ff5513004bbb160e93af77ad278a0da Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 18 Aug 2023 20:29:18 -0700 Subject: [PATCH 07/29] Remove usused field --- dagsync/ipnisync/publisher.go | 1 - 1 file changed, 1 deletion(-) diff --git a/dagsync/ipnisync/publisher.go b/dagsync/ipnisync/publisher.go index 88fdbbf..6da9d68 100644 --- a/dagsync/ipnisync/publisher.go +++ b/dagsync/ipnisync/publisher.go @@ -24,7 +24,6 @@ import ( // Publisher serves an advertisement chain over HTTP. type Publisher struct { - addr multiaddr.Multiaddr lsys ipld.LinkSystem handlerPath string peerID peer.ID From 696de20c055ba7f2eb703f3545fb5b2c75c44a29 Mon Sep 17 00:00:00 2001 From: gammazero Date: Sat, 19 Aug 2023 03:01:48 -0700 Subject: [PATCH 08/29] Sync gets addrs from peerstore if none supplied --- dagsync/ipnisync/sync.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index 5a91df8..42c907d 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -114,6 +114,20 @@ func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { httpClient = rclient.StandardClient() } + if len(peerInfo.Addrs) == 0 { + if s.clientHost.StreamHost == nil { + return nil, errors.New("no peer addrs and no stream host") + } + peerStore := s.clientHost.StreamHost.Peerstore() + if peerStore == nil { + return nil, errors.New("no peer addrs and no stream host peerstore") + } + peerInfo.Addrs = peerStore.Addrs(peerInfo.ID) + if len(peerInfo.Addrs) == 0 { + return nil, errors.New("no peer addrs and none found in peertore") + } + } + urls := make([]*url.URL, len(peerInfo.Addrs)) for i, addr := range peerInfo.Addrs { u, err := maurl.ToURL(addr) From 6fae9ffa32e45d315f726a2d1827153cceb2bf15 Mon Sep 17 00:00:00 2001 From: gammazero Date: Sat, 19 Aug 2023 03:22:55 -0700 Subject: [PATCH 09/29] Use dtsync if publisher server if libp2p without HTTP --- dagsync/ipnisync/sync.go | 6 ++++++ dagsync/subscriber.go | 12 ++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index 42c907d..098cab5 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "strings" "time" "github.com/hashicorp/go-retryablehttp" @@ -31,6 +32,8 @@ const ProtocolID = protocol.ID("/ipnisync/v1") var log = logging.Logger("dagsync/ipnisync") +var ErrNoHTTPServer = errors.New("publisher has libp2p server without HTTP") + // Sync provides sync functionality for use with all http syncs. type Sync struct { blockHook func(peer.ID, cid.Cid) @@ -94,6 +97,9 @@ func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { cli, err = s.clientHost.NamespacedClient(ProtocolID, peerInfo) } if err != nil { + if strings.Contains(err.Error(), "failed to negotiate protocol: protocols not supported") { + return nil, ErrNoHTTPServer + } log.Warnw("Cannot create libp2phttp client. Server is not a libp2phttp server. Using plain http", "err", err) httpClient = &s.client } else { diff --git a/dagsync/subscriber.go b/dagsync/subscriber.go index f2f47a8..4651187 100644 --- a/dagsync/subscriber.go +++ b/dagsync/subscriber.go @@ -804,8 +804,16 @@ func (s *Subscriber) makeSyncer(peerInfo peer.AddrInfo, doUpdate bool) (Syncer, } } - // Not an httpPeerAddr, so use the dtSync. - return s.dtSync.NewSyncer(peerInfo.ID, s.topicName), update, nil + syncer, err := s.ipniSync.NewSyncer(peerInfo) + if err != nil { + if errors.Is(err, ipnisync.ErrNoHTTPServer) { + log.Warn(err.Error()) + // Publisher is libp2p without HTTP, so use the dtSync. + return s.dtSync.NewSyncer(peerInfo.ID, s.topicName), update, nil + } + return nil, nil, fmt.Errorf("cannot create ipni-sync handler: %w", err) + } + return syncer, update, nil } // asyncSyncAdChain processes the latest announce message received over pubsub From d7b76b3b1bbf0fb0d75e2eee752b147ad740aee8 Mon Sep 17 00:00:00 2001 From: gammazero Date: Sat, 19 Aug 2023 18:40:19 -0700 Subject: [PATCH 10/29] If server no libp2phttp, then use address to choose plain HTTP or dtsync. --- dagsync/ipnisync/sync.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index 098cab5..55fa8c5 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -22,6 +22,7 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector" headschema "github.com/ipni/go-libipni/dagsync/ipnisync/head" "github.com/ipni/go-libipni/maurl" + "github.com/ipni/go-libipni/mautil" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" libp2phttp "github.com/libp2p/go-libp2p/p2p/http" @@ -97,7 +98,11 @@ func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { cli, err = s.clientHost.NamespacedClient(ProtocolID, peerInfo) } if err != nil { - if strings.Contains(err.Error(), "failed to negotiate protocol: protocols not supported") { + if !strings.Contains(err.Error(), "failed to negotiate protocol: protocols not supported") { + return nil, err + } + httpAddrs := mautil.FindHTTPAddrs(peerInfo.Addrs) + if len(httpAddrs) == 0 { return nil, ErrNoHTTPServer } log.Warnw("Cannot create libp2phttp client. Server is not a libp2phttp server. Using plain http", "err", err) From 03da4f6d432612b2e78da4854459ea0c4c354034 Mon Sep 17 00:00:00 2001 From: gammazero Date: Sat, 19 Aug 2023 19:36:59 -0700 Subject: [PATCH 11/29] Fix error handling in syncer creation --- dagsync/ipnisync/sync.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index 55fa8c5..2310aa5 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -8,7 +8,6 @@ import ( "io" "net/http" "net/url" - "strings" "time" "github.com/hashicorp/go-retryablehttp" @@ -98,9 +97,6 @@ func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { cli, err = s.clientHost.NamespacedClient(ProtocolID, peerInfo) } if err != nil { - if !strings.Contains(err.Error(), "failed to negotiate protocol: protocols not supported") { - return nil, err - } httpAddrs := mautil.FindHTTPAddrs(peerInfo.Addrs) if len(httpAddrs) == 0 { return nil, ErrNoHTTPServer From 62961add3d9c2a3d1a1a80857da7049e5bc78046 Mon Sep 17 00:00:00 2001 From: gammazero Date: Sat, 19 Aug 2023 20:24:01 -0700 Subject: [PATCH 12/29] Fix race when accessing libp2phttp.HTTPHost --- dagsync/ipnisync/sync.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index 2310aa5..652729e 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "sync" "time" "github.com/hashicorp/go-retryablehttp" @@ -42,9 +43,10 @@ type Sync struct { httpTimeout time.Duration // libp2phttp - clientHost *libp2phttp.HTTPHost - authPeerID bool - rclient *retryablehttp.Client + clientHost *libp2phttp.HTTPHost + clientHostMutex sync.Mutex + authPeerID bool + rclient *retryablehttp.Client } // Syncer provides sync functionality for a single sync with a peer. @@ -91,11 +93,13 @@ func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { var cli http.Client var httpClient *http.Client var err error + s.clientHostMutex.Lock() if s.authPeerID { cli, err = s.clientHost.NamespacedClient(ProtocolID, peerInfo, libp2phttp.ServerMustAuthenticatePeerID) } else { cli, err = s.clientHost.NamespacedClient(ProtocolID, peerInfo) } + s.clientHostMutex.Unlock() if err != nil { httpAddrs := mautil.FindHTTPAddrs(peerInfo.Addrs) if len(httpAddrs) == 0 { @@ -155,9 +159,9 @@ func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { func (s *Sync) Close() { s.client.CloseIdleConnections() - if s.clientHost != nil { - s.clientHost.Close() - } + s.clientHostMutex.Lock() + s.clientHost.Close() + s.clientHostMutex.Unlock() } // GetHead fetches the head of the peer's advertisement chain. From ab495c8134feb1b080f30f187fbb5b7dbb281057 Mon Sep 17 00:00:00 2001 From: gammazero Date: Mon, 21 Aug 2023 09:20:56 -0700 Subject: [PATCH 13/29] If publisher HTTP no availabe at IPNI path, retry without IPNI path. This supports legacy HTTP served without IPNI path. --- dagsync/ipnisync/sync.go | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index 652729e..090d89b 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "strings" "sync" "time" @@ -56,6 +57,10 @@ type Syncer struct { rootURL url.URL urls []*url.URL sync *Sync + + // For legacy HTTP and external server support without IPNI path. + noPath bool + plainHTTP bool } // NewSync creates a new Sync. @@ -100,6 +105,7 @@ func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { cli, err = s.clientHost.NamespacedClient(ProtocolID, peerInfo) } s.clientHostMutex.Unlock() + var plainHTTP bool if err != nil { httpAddrs := mautil.FindHTTPAddrs(peerInfo.Addrs) if len(httpAddrs) == 0 { @@ -107,6 +113,7 @@ func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { } log.Warnw("Cannot create libp2phttp client. Server is not a libp2phttp server. Using plain http", "err", err) httpClient = &s.client + plainHTTP = true } else { httpClient = &cli } @@ -154,6 +161,8 @@ func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { rootURL: *urls[0], urls: urls[1:], sync: s, + + plainHTTP: plainHTTP, }, nil } @@ -282,7 +291,6 @@ func (s *Syncer) walkFetch(ctx context.Context, rootCid cid.Cid, sel selector.Se func (s *Syncer) fetch(ctx context.Context, rsrc string, cb func(io.Reader) error) error { nextURL: fetchURL := s.rootURL.JoinPath(rsrc) - req, err := http.NewRequestWithContext(ctx, "GET", fetchURL.String(), nil) if err != nil { return err @@ -294,6 +302,9 @@ nextURL: log.Errorw("Fetch request failed, will retry with next address", "err", err) s.rootURL = *s.urls[0] s.urls = s.urls[1:] + if s.noPath { + s.rootURL.Path = strings.TrimSuffix(s.rootURL.Path, strings.Trim(IPNIPath, "/")) + } goto nextURL } return fmt.Errorf("fetch request failed: %w", err) @@ -301,15 +312,31 @@ nextURL: defer resp.Body.Close() switch resp.StatusCode { + case http.StatusOK: + log.Debugw("Found block from HTTP publisher", "resource", rsrc) + return cb(resp.Body) case http.StatusNotFound: + if s.plainHTTP && !s.noPath { + // Try again with no path for legacy http. + log.Warnw("Plain HTTP got not found response, retrying without IPNI path for legacy HTTP") + s.rootURL.Path = strings.TrimSuffix(s.rootURL.Path, strings.Trim(IPNIPath, "/")) + s.noPath = true + goto nextURL + } log.Errorw("Block not found from HTTP publisher", "resource", rsrc) // Include the string "content not found" so that indexers that have not // upgraded gracefully handle the error case. Because, this string is // being checked already. return fmt.Errorf("content not found: %w", ipld.ErrNotExists{}) - case http.StatusOK: - log.Debugw("Found block from HTTP publisher", "resource", rsrc) - return cb(resp.Body) + case http.StatusForbidden: + if s.plainHTTP && !s.noPath { + // Try again with no path for legacy http. + log.Warnw("Plain HTTP got forbidden response, retrying without IPNI path for legacy HTTP") + s.rootURL.Path = strings.TrimSuffix(s.rootURL.Path, strings.Trim(IPNIPath, "/")) + s.noPath = true + goto nextURL + } + fallthrough default: return fmt.Errorf("non success http fetch response at %s: %d", fetchURL.String(), resp.StatusCode) } From ef384109d5539448363b527252e02774272b2664 Mon Sep 17 00:00:00 2001 From: gammazero Date: Mon, 21 Aug 2023 14:40:50 -0700 Subject: [PATCH 14/29] Use latest libp2phttp --- dagsync/ipnisync/publisher.go | 4 ++-- dagsync/ipnisync/sync.go | 12 ++++++------ go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dagsync/ipnisync/publisher.go b/dagsync/ipnisync/publisher.go index 6da9d68..0d45290 100644 --- a/dagsync/ipnisync/publisher.go +++ b/dagsync/ipnisync/publisher.go @@ -32,7 +32,7 @@ type Publisher struct { root cid.Cid topic string - pubHost *libp2phttp.HTTPHost + pubHost *libp2phttp.Host // httpAddrs is returned by Addrs when not starting the server. httpAddrs []multiaddr.Multiaddr } @@ -101,7 +101,7 @@ func NewPublisher(lsys ipld.LinkSystem, privKey ic.PrivKey, options ...Option) ( // This is the "HTTP Host". It's like the libp2p "stream host" (aka core // host.Host), but it uses HTTP semantics instead of stream semantics. - publisherHost := &libp2phttp.HTTPHost{ + publisherHost := &libp2phttp.Host{ StreamHost: opts.streamHost, ListenAddrs: httpListenAddrs, ServeInsecureHTTP: !opts.requireTLS, diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index 090d89b..c261fc1 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -44,7 +44,7 @@ type Sync struct { httpTimeout time.Duration // libp2phttp - clientHost *libp2phttp.HTTPHost + clientHost *libp2phttp.Host clientHostMutex sync.Mutex authPeerID bool rclient *retryablehttp.Client @@ -72,7 +72,7 @@ func NewSync(lsys ipld.LinkSystem, blockHook func(peer.ID, cid.Cid), options ... client: http.Client{ Timeout: opts.httpTimeout, }, - clientHost: &libp2phttp.HTTPHost{ + clientHost: &libp2phttp.Host{ StreamHost: opts.streamHost, }, lsys: lsys, @@ -98,12 +98,12 @@ func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { var cli http.Client var httpClient *http.Client var err error - s.clientHostMutex.Lock() + var rtOpts []libp2phttp.RoundTripperOption if s.authPeerID { - cli, err = s.clientHost.NamespacedClient(ProtocolID, peerInfo, libp2phttp.ServerMustAuthenticatePeerID) - } else { - cli, err = s.clientHost.NamespacedClient(ProtocolID, peerInfo) + rtOpts = append(rtOpts, libp2phttp.ServerMustAuthenticatePeerID) } + s.clientHostMutex.Lock() + cli, err = s.clientHost.NamespacedClient(ProtocolID, peerInfo, rtOpts...) s.clientHostMutex.Unlock() var plainHTTP bool if err != nil { diff --git a/go.mod b/go.mod index fcf010c..56521f5 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/ipfs/go-ipld-format v0.3.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-ipld-prime v0.20.0 - github.com/libp2p/go-libp2p v0.29.1-0.20230816203758-9566227604f8 + github.com/libp2p/go-libp2p v0.29.1-0.20230821201003-23bc261aa509 github.com/libp2p/go-libp2p-gostream v0.6.0 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-msgio v0.3.0 diff --git a/go.sum b/go.sum index 3f9f9d1..0a42d76 100644 --- a/go.sum +++ b/go.sum @@ -305,8 +305,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.29.1-0.20230816203758-9566227604f8 h1:nR6nNb5b/f4AF2VdDHZBzWsi/5Gppb6raiHwyKNXKd0= -github.com/libp2p/go-libp2p v0.29.1-0.20230816203758-9566227604f8/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= +github.com/libp2p/go-libp2p v0.29.1-0.20230821201003-23bc261aa509 h1:UKZTdF3iBeQ9SapDqx2x2oq31Nw+SV6ej/g6Jtckkt8= +github.com/libp2p/go-libp2p v0.29.1-0.20230821201003-23bc261aa509/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU= From faa5859b4bc975a1105602dd3a35e01b841ad9e0 Mon Sep 17 00:00:00 2001 From: gammazero Date: Mon, 21 Aug 2023 16:32:16 -0700 Subject: [PATCH 15/29] latest libp2phttp --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 56521f5..6288554 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/ipfs/go-ipld-format v0.3.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-ipld-prime v0.20.0 - github.com/libp2p/go-libp2p v0.29.1-0.20230821201003-23bc261aa509 + github.com/libp2p/go-libp2p v0.29.1-0.20230821203856-d6b9f7d05dd3 github.com/libp2p/go-libp2p-gostream v0.6.0 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-msgio v0.3.0 diff --git a/go.sum b/go.sum index 0a42d76..5351889 100644 --- a/go.sum +++ b/go.sum @@ -305,8 +305,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.29.1-0.20230821201003-23bc261aa509 h1:UKZTdF3iBeQ9SapDqx2x2oq31Nw+SV6ej/g6Jtckkt8= -github.com/libp2p/go-libp2p v0.29.1-0.20230821201003-23bc261aa509/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= +github.com/libp2p/go-libp2p v0.29.1-0.20230821203856-d6b9f7d05dd3 h1:y0xVN2nnnhfoDVz2Lbd/krA1AHBQulLxKjF7AC0SBss= +github.com/libp2p/go-libp2p v0.29.1-0.20230821203856-d6b9f7d05dd3/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU= From f1e326359d4aadb55d19e71e6503a8213b77030a Mon Sep 17 00:00:00 2001 From: gammazero Date: Wed, 23 Aug 2023 00:41:40 -0700 Subject: [PATCH 16/29] ignore emtpy http listen addr --- dagsync/ipnisync/option.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dagsync/ipnisync/option.go b/dagsync/ipnisync/option.go index ce17645..dcc6be8 100644 --- a/dagsync/ipnisync/option.go +++ b/dagsync/ipnisync/option.go @@ -43,7 +43,11 @@ func getOpts(opts []Option) (config, error) { // the WithStreamHost option. func WithHTTPListenAddrs(addrs ...string) Option { return func(c *config) error { - c.httpAddrs = append(c.httpAddrs, addrs...) + for _, addr := range addrs { + if addr != "" { + c.httpAddrs = append(c.httpAddrs, addr) + } + } return nil } } From 0669ee8d4bbed30c47836f3b2667de452e281e68 Mon Sep 17 00:00:00 2001 From: gammazero Date: Wed, 23 Aug 2023 20:06:25 -0700 Subject: [PATCH 17/29] update to latest libp2phttp --- dagsync/ipnisync/publisher.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dagsync/ipnisync/publisher.go b/dagsync/ipnisync/publisher.go index 0d45290..601751f 100644 --- a/dagsync/ipnisync/publisher.go +++ b/dagsync/ipnisync/publisher.go @@ -104,7 +104,7 @@ func NewPublisher(lsys ipld.LinkSystem, privKey ic.PrivKey, options ...Option) ( publisherHost := &libp2phttp.Host{ StreamHost: opts.streamHost, ListenAddrs: httpListenAddrs, - ServeInsecureHTTP: !opts.requireTLS, + InsecureAllowHTTP: !opts.requireTLS, } pub.pubHost = publisherHost diff --git a/go.mod b/go.mod index 6288554..a2326bf 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/ipfs/go-ipld-format v0.3.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-ipld-prime v0.20.0 - github.com/libp2p/go-libp2p v0.29.1-0.20230821203856-d6b9f7d05dd3 + github.com/libp2p/go-libp2p v0.29.1-0.20230823171643-912f92d49a19 github.com/libp2p/go-libp2p-gostream v0.6.0 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-msgio v0.3.0 diff --git a/go.sum b/go.sum index 5351889..8f5f564 100644 --- a/go.sum +++ b/go.sum @@ -305,8 +305,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.29.1-0.20230821203856-d6b9f7d05dd3 h1:y0xVN2nnnhfoDVz2Lbd/krA1AHBQulLxKjF7AC0SBss= -github.com/libp2p/go-libp2p v0.29.1-0.20230821203856-d6b9f7d05dd3/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= +github.com/libp2p/go-libp2p v0.29.1-0.20230823171643-912f92d49a19 h1:oBCMCeLFvvHhAsLy9/9ybP71Sc5/DSGtZKn9aSLrwwk= +github.com/libp2p/go-libp2p v0.29.1-0.20230823171643-912f92d49a19/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU= From 999067b063137ae78e806bfdde8a050a0fb35972 Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 24 Aug 2023 13:52:21 -0700 Subject: [PATCH 18/29] Update log messages --- dagsync/ipnisync/sync.go | 3 ++- dagsync/subscriber.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index c261fc1..b212194 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -111,10 +111,11 @@ func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { if len(httpAddrs) == 0 { return nil, ErrNoHTTPServer } - log.Warnw("Cannot create libp2phttp client. Server is not a libp2phttp server. Using plain http", "err", err) + log.Infow("Publisher is not a libp2phttp server. Using plain http", "err", err, "publisher", peerInfo.ID) httpClient = &s.client plainHTTP = true } else { + log.Infow("Publisher supports libp2phttp", "publisher", peerInfo.ID) httpClient = &cli } httpClient.Timeout = s.httpTimeout diff --git a/dagsync/subscriber.go b/dagsync/subscriber.go index 4651187..b2f77d3 100644 --- a/dagsync/subscriber.go +++ b/dagsync/subscriber.go @@ -807,7 +807,7 @@ func (s *Subscriber) makeSyncer(peerInfo peer.AddrInfo, doUpdate bool) (Syncer, syncer, err := s.ipniSync.NewSyncer(peerInfo) if err != nil { if errors.Is(err, ipnisync.ErrNoHTTPServer) { - log.Warn(err.Error()) + log.Warnf("Using data-transfer sync with %s: %s", peerInfo.ID, err.Error()) // Publisher is libp2p without HTTP, so use the dtSync. return s.dtSync.NewSyncer(peerInfo.ID, s.topicName), update, nil } From 309ba518c7b212c62805b3973cd5db2057e8788b Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 24 Aug 2023 15:36:08 -0700 Subject: [PATCH 19/29] Log peer.ID consistently as peer --- dagsync/ipnisync/sync.go | 7 +++---- dagsync/subscriber.go | 21 ++++++++++----------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index b212194..ffa7312 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -111,11 +111,11 @@ func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { if len(httpAddrs) == 0 { return nil, ErrNoHTTPServer } - log.Infow("Publisher is not a libp2phttp server. Using plain http", "err", err, "publisher", peerInfo.ID) + log.Infow("Publisher is not a libp2phttp server. Using plain http", "err", err, "peer", peerInfo.ID) httpClient = &s.client plainHTTP = true } else { - log.Infow("Publisher supports libp2phttp", "publisher", peerInfo.ID) + log.Infow("Publisher supports libp2phttp", "peer", peerInfo.ID) httpClient = &cli } httpClient.Timeout = s.httpTimeout @@ -191,7 +191,7 @@ func (s *Syncer) GetHead(ctx context.Context) (cid.Cid, error) { return cid.Undef, err } if s.peerID == "" { - log.Warn("cannot verify publisher signature without peer ID") + log.Warn("Cannot verify publisher signature without peer ID") } else if signerID != s.peerID { return cid.Undef, errors.New("found head signed by an unexpected peer") } @@ -314,7 +314,6 @@ nextURL: switch resp.StatusCode { case http.StatusOK: - log.Debugw("Found block from HTTP publisher", "resource", rsrc) return cb(resp.Body) case http.StatusNotFound: if s.plainHTTP && !s.noPath { diff --git a/dagsync/subscriber.go b/dagsync/subscriber.go index b2f77d3..9746e5e 100644 --- a/dagsync/subscriber.go +++ b/dagsync/subscriber.go @@ -377,7 +377,7 @@ func (s *Subscriber) RemoveHandler(peerID peer.ID) bool { return false } - log.Infow("Removing handler for publisher", "peer", peerID) + log.Infow("Removing sync handler", "peer", peerID) delete(s.handlers, peerID) return true @@ -554,7 +554,7 @@ func (s *Subscriber) SyncHAMTEntries(ctx context.Context, peerInfo peer.AddrInfo func (s *Subscriber) syncEntries(ctx context.Context, peerInfo peer.AddrInfo, entCid cid.Cid, sel ipld.Node, bh BlockHookFunc, segdl int64) error { if entCid == cid.Undef { - log.Info("No entries to sync") + log.Info("No entries to sync", "peer", peerInfo.ID) return nil } @@ -577,8 +577,7 @@ func (s *Subscriber) syncEntries(ctx context.Context, peerInfo peer.AddrInfo, en return err } - log := log.With("peer", peerInfo.ID, "cid", entCid) - log.Info("Start entries sync sync") + log.Info("Start entries sync", "peer", peerInfo.ID, "cid", entCid) // Check for an existing handler for the specified peer (publisher). If // none, create one if allowed. @@ -681,7 +680,7 @@ func (s *Subscriber) idleHandlerCleaner() { for pid, hnd := range s.handlers { if now.After(hnd.expires) { delete(s.handlers, pid) - log.Debugw("Removed idle handler", "publisherID", pid) + log.Debugw("Removed idle handler", "peer", pid) } } s.handlersMutex.Unlock() @@ -716,7 +715,7 @@ func (s *Subscriber) watch() { // If rhw previous pending message was not nil, then there is an // existing request to sync the ad chain. if oldMsg != nil { - log.Infow("Pending announce replaced by new", "previous_cid", oldMsg.Cid, "new_cid", amsg.Cid, "publisher", hnd.peerID) + log.Infow("Pending announce replaced by new", "previous_cid", oldMsg.Cid, "new_cid", amsg.Cid, "peer", hnd.peerID) continue } @@ -807,7 +806,7 @@ func (s *Subscriber) makeSyncer(peerInfo peer.AddrInfo, doUpdate bool) (Syncer, syncer, err := s.ipniSync.NewSyncer(peerInfo) if err != nil { if errors.Is(err, ipnisync.ErrNoHTTPServer) { - log.Warnf("Using data-transfer sync with %s: %s", peerInfo.ID, err.Error()) + log.Warnw("Using data-transfer sync", "peer", peerInfo.ID, "reason", err.Error()) // Publisher is libp2p without HTTP, so use the dtSync. return s.dtSync.NewSyncer(peerInfo.ID, s.topicName), update, nil } @@ -821,7 +820,7 @@ func (s *Subscriber) makeSyncer(peerInfo peer.AddrInfo, doUpdate bool) (Syncer, // one goroutine per advertisement publisher. func (h *handler) asyncSyncAdChain(ctx context.Context) { if ctx.Err() != nil { - log.Warnw("Abandoned pending sync", "err", ctx.Err(), "publisher", h.peerID) + log.Warnw("Abandoned pending sync", "err", ctx.Err(), "peer", h.peerID) return } @@ -833,7 +832,7 @@ func (h *handler) asyncSyncAdChain(ctx context.Context) { } syncer, updatePeerstore, err := h.subscriber.makeSyncer(peerInfo, true) if err != nil { - log.Errorw("Cannot make syncer for announce", "err", err) + log.Errorw("Cannot make syncer for announce", "err", err, "peer", h.peerID) return } @@ -843,7 +842,7 @@ func (h *handler) asyncSyncAdChain(ctx context.Context) { if latestSyncLink != nil { stopAtCid = latestSyncLink.(cidlink.Link).Cid if stopAtCid == nextCid { - log.Infow("cid to sync to is the stop node. Nothing to do") + log.Infow("CID to sync to is the stop node. Nothing to do.", "peer", h.peerID) return } } @@ -855,7 +854,7 @@ func (h *handler) asyncSyncAdChain(ctx context.Context) { if h.subscriber.receiver != nil { h.subscriber.receiver.UncacheCid(nextCid) } - log.Errorw("Cannot process message", "err", err, "publisher", h.peerID) + log.Errorw("Cannot process message", "err", err, "peer", h.peerID) if strings.Contains(err.Error(), "response rejected") { // A "response rejected" error happens when the indexer does not // allow a provider. This is not an error with provider, so do not From 1e89b8619b201c60adf6fedff749dd2b50c9ffec Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 24 Aug 2023 17:56:38 -0700 Subject: [PATCH 20/29] Update libp2p --- dagsync/ipnisync/sync.go | 15 +++++++-------- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index ffa7312..e71a89f 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -267,6 +267,12 @@ func (s *Syncer) walkFetch(ctx context.Context, rootCid cid.Cid, sel selector.Se return r, err } + // get the direct node. + rootNode, err := getMissingLs.Load(ipld.LinkContext{Ctx: ctx}, cidlink.Link{Cid: rootCid}, basicnode.Prototype.Any) + if err != nil { + return nil, fmt.Errorf("failed to load node for root cid %s: %w", rootCid, err) + } + progress := traversal.Progress{ Cfg: &traversal.Config{ Ctx: ctx, @@ -275,14 +281,7 @@ func (s *Syncer) walkFetch(ctx context.Context, rootCid cid.Cid, sel selector.Se }, Path: datamodel.NewPath([]datamodel.PathSegment{}), } - // get the direct node. - rootNode, err := getMissingLs.Load(ipld.LinkContext{Ctx: ctx}, cidlink.Link{Cid: rootCid}, basicnode.Prototype.Any) - if err != nil { - return nil, fmt.Errorf("failed to load node for root cid %s: %w", rootCid, err) - } - err = progress.WalkMatching(rootNode, sel, func(p traversal.Progress, n datamodel.Node) error { - return nil - }) + err = progress.WalkMatching(rootNode, sel, func(_ traversal.Progress, _ datamodel.Node) error { return nil }) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index a2326bf..27f9061 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/ipfs/go-ipld-format v0.3.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-ipld-prime v0.20.0 - github.com/libp2p/go-libp2p v0.29.1-0.20230823171643-912f92d49a19 + github.com/libp2p/go-libp2p v0.29.1-0.20230824185545-1af5d3baeec0 github.com/libp2p/go-libp2p-gostream v0.6.0 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-msgio v0.3.0 diff --git a/go.sum b/go.sum index 8f5f564..1d77e3e 100644 --- a/go.sum +++ b/go.sum @@ -305,8 +305,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.29.1-0.20230823171643-912f92d49a19 h1:oBCMCeLFvvHhAsLy9/9ybP71Sc5/DSGtZKn9aSLrwwk= -github.com/libp2p/go-libp2p v0.29.1-0.20230823171643-912f92d49a19/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= +github.com/libp2p/go-libp2p v0.29.1-0.20230824185545-1af5d3baeec0 h1:qaUGo0lQDfzTCsdnPFDGCNbYAq9tJvhUL+c/WzPgetg= +github.com/libp2p/go-libp2p v0.29.1-0.20230824185545-1af5d3baeec0/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU= From 352032b6833189b4aa78f20403cd6a5014d30764 Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 25 Aug 2023 11:43:11 -0700 Subject: [PATCH 21/29] Convert most tests to use ipnisync publisher - Change AsyncErr to Err in SyncFinished - Move old p2p head client/server (legs protocol ID) into dtsync, since that is the only place it is used. - Add tests --- dagsync/announce_test.go | 53 ++++--- dagsync/{p2p/protocol => dtsync}/head/head.go | 0 .../protocol => dtsync}/head/head_test.go | 2 +- dagsync/dtsync/publisher.go | 2 +- dagsync/dtsync/syncer.go | 2 +- dagsync/example_test.go | 13 +- dagsync/subscriber.go | 12 +- dagsync/subscriber_test.go | 40 +++--- dagsync/sync_test.go | 136 +++++++++++++----- dagsync/test/util.go | 10 +- 10 files changed, 169 insertions(+), 101 deletions(-) rename dagsync/{p2p/protocol => dtsync}/head/head.go (100%) rename dagsync/{p2p/protocol => dtsync}/head/head_test.go (98%) diff --git a/dagsync/announce_test.go b/dagsync/announce_test.go index 7313a6f..b262be5 100644 --- a/dagsync/announce_test.go +++ b/dagsync/announce_test.go @@ -27,36 +27,29 @@ import ( func TestAnnounceReplace(t *testing.T) { t.Parallel() - srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost := test.MkTestHost(t) - srcHostInfo := peer.AddrInfo{ - ID: srcHost.ID(), - Addrs: srcHost.Addrs(), - } - srcLnkS := test.MkLinkSystem(srcStore) dstHost := test.MkTestHost(t) - - srcHost.Peerstore().AddAddrs(dstHost.ID(), dstHost.Addrs(), time.Hour) - dstHost.Peerstore().AddAddrs(srcHost.ID(), srcHost.Addrs(), time.Hour) - //dstLnkS := test.MkLinkSystem(dstStore) - dstLnkS, blocked := test.MkBlockedLinkSystem(dstStore) blocksSeenByHook := make(map[cid.Cid]struct{}) blockHook := func(p peer.ID, c cid.Cid, _ dagsync.SegmentSyncActions) { blocksSeenByHook[c] = struct{}{} } - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) - require.NoError(t, err) - defer pub.Close() - sub, err := dagsync.NewSubscriber(dstHost, dstStore, dstLnkS, testTopic, dagsync.RecvAnnounce(), dagsync.BlockHook(blockHook)) require.NoError(t, err) defer sub.Close() - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, testTopic)) + srcHost, srcPrivKey := test.MkTestHostPK(t) + srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) + srcLnkS := test.MkLinkSystem(srcStore) + + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost), ipnisync.WithHeadTopic(testTopic)) + require.NoError(t, err) + defer pub.Close() + + srcHost.Peerstore().AddAddrs(dstHost.ID(), dstHost.Addrs(), time.Hour) + dstHost.Peerstore().AddAddrs(srcHost.ID(), srcHost.Addrs(), time.Hour) watcher, cncl := sub.OnSyncFinished() defer cncl() @@ -67,6 +60,11 @@ func TestAnnounceReplace(t *testing.T) { firstCid := chainLnks[2].(cidlink.Link).Cid pub.SetRoot(firstCid) + srcHostInfo := peer.AddrInfo{ + ID: srcHost.ID(), + Addrs: srcHost.Addrs(), + } + // Have the subscriber receive an announce. This is the same as if it was // published by the publisher without having to wait for it to arrive. err = sub.Announce(context.Background(), firstCid, srcHostInfo) @@ -214,11 +212,7 @@ func TestAnnounce_LearnsHttpPublisherAddr(t *testing.T) { func TestAnnounceRepublish(t *testing.T) { srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost := test.MkTestHost(t) - srcHostInfo := peer.AddrInfo{ - ID: srcHost.ID(), - Addrs: srcHost.Addrs(), - } + srcHost, srcPrivKey := test.MkTestHostPK(t) srcLnkS := test.MkLinkSystem(srcStore) dstHost := test.MkTestHost(t) @@ -243,10 +237,9 @@ func TestAnnounceRepublish(t *testing.T) { require.NoError(t, err) defer sub1.Close() - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost), ipnisync.WithHeadTopic(testTopic)) require.NoError(t, err) defer pub.Close() - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, testTopic)) watcher2, cncl := sub2.OnSyncFinished() defer cncl() @@ -258,7 +251,11 @@ func TestAnnounceRepublish(t *testing.T) { pub.SetRoot(firstCid) // Announce one CID to subscriber1. - err = sub1.Announce(context.Background(), firstCid, srcHostInfo) + pubInfo := peer.AddrInfo{ + ID: pub.ID(), + Addrs: pub.Addrs(), + } + err = sub1.Announce(context.Background(), firstCid, pubInfo) require.NoError(t, err) t.Log("Sent announce for first CID", firstCid) @@ -444,7 +441,7 @@ func mkLnk(t *testing.T, srcStore datastore.Batching) cid.Cid { } func initPubSub(t *testing.T, srcStore, dstStore datastore.Batching, allowPeer func(peer.ID) bool) (host.Host, host.Host, dagsync.Publisher, *dagsync.Subscriber, announce.Sender) { - srcHost := test.MkTestHost(t) + srcHost, srcPrivKey := test.MkTestHostPK(t) dstHost := test.MkTestHost(t) topics := test.WaitForMeshWithMessage(t, testTopic, srcHost, dstHost) @@ -453,7 +450,7 @@ func initPubSub(t *testing.T, srcStore, dstStore datastore.Batching, allowPeer f p2pSender, err := p2psender.New(nil, "", p2psender.WithTopic(topics[0]), p2psender.WithExtraData([]byte("t01000"))) require.NoError(t, err) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost), ipnisync.WithHeadTopic(testTopic)) require.NoError(t, err) srcHost.Peerstore().AddAddrs(dstHost.ID(), dstHost.Addrs(), time.Hour) @@ -467,7 +464,5 @@ func initPubSub(t *testing.T, srcStore, dstStore datastore.Batching, allowPeer f err = srcHost.Connect(context.Background(), dstHost.Peerstore().PeerInfo(dstHost.ID())) require.NoError(t, err) - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, testTopic)) - return srcHost, dstHost, pub, sub, p2pSender } diff --git a/dagsync/p2p/protocol/head/head.go b/dagsync/dtsync/head/head.go similarity index 100% rename from dagsync/p2p/protocol/head/head.go rename to dagsync/dtsync/head/head.go diff --git a/dagsync/p2p/protocol/head/head_test.go b/dagsync/dtsync/head/head_test.go similarity index 98% rename from dagsync/p2p/protocol/head/head_test.go rename to dagsync/dtsync/head/head_test.go index d136309..6c7acc5 100644 --- a/dagsync/p2p/protocol/head/head_test.go +++ b/dagsync/dtsync/head/head_test.go @@ -12,7 +12,7 @@ import ( _ "github.com/ipld/go-ipld-prime/codec/dagjson" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" - "github.com/ipni/go-libipni/dagsync/p2p/protocol/head" + "github.com/ipni/go-libipni/dagsync/dtsync/head" "github.com/ipni/go-libipni/dagsync/test" "github.com/libp2p/go-libp2p/core/protocol" "github.com/multiformats/go-multiaddr" diff --git a/dagsync/dtsync/publisher.go b/dagsync/dtsync/publisher.go index 4546099..692778a 100644 --- a/dagsync/dtsync/publisher.go +++ b/dagsync/dtsync/publisher.go @@ -11,7 +11,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipld/go-ipld-prime" - "github.com/ipni/go-libipni/dagsync/p2p/protocol/head" + "github.com/ipni/go-libipni/dagsync/dtsync/head" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" diff --git a/dagsync/dtsync/syncer.go b/dagsync/dtsync/syncer.go index 0b1c1d7..e8a8141 100644 --- a/dagsync/dtsync/syncer.go +++ b/dagsync/dtsync/syncer.go @@ -12,7 +12,7 @@ import ( "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/traversal" "github.com/ipld/go-ipld-prime/traversal/selector" - "github.com/ipni/go-libipni/dagsync/p2p/protocol/head" + "github.com/ipni/go-libipni/dagsync/dtsync/head" "github.com/libp2p/go-libp2p/core/peer" ) diff --git a/dagsync/example_test.go b/dagsync/example_test.go index ced5850..1dfb0db 100644 --- a/dagsync/example_test.go +++ b/dagsync/example_test.go @@ -3,6 +3,7 @@ package dagsync_test import ( "bytes" "context" + "crypto/rand" "fmt" "io" "log" @@ -15,8 +16,9 @@ import ( cidlink "github.com/ipld/go-ipld-prime/linking/cid" basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/ipni/go-libipni/dagsync" - "github.com/ipni/go-libipni/dagsync/dtsync" + "github.com/ipni/go-libipni/dagsync/ipnisync" "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/host" "github.com/multiformats/go-multicodec" ) @@ -25,12 +27,13 @@ var srcHost host.Host func ExamplePublisher() { // Init dagsync publisher and subscriber. - srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost, _ = libp2p.New() + srcPrivKey, _, _ := crypto.GenerateEd25519Key(rand.Reader) + srcHost, _ = libp2p.New(libp2p.Identity(srcPrivKey)) defer srcHost.Close() + srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) srcLnkS := makeLinkSystem(srcStore) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost), ipnisync.WithHeadTopic("/indexer/ingest/testnet")) if err != nil { panic(err) } @@ -65,7 +68,7 @@ func ExampleSubscriber() { srcHost.Peerstore().AddAddrs(dstHost.ID(), dstHost.Addrs(), time.Hour) dstHost.Peerstore().AddAddrs(srcHost.ID(), srcHost.Addrs(), time.Hour) - sub, err := dagsync.NewSubscriber(dstHost, dstStore, dstLnkSys, "/indexer/ingest/testnet", nil) + sub, err := dagsync.NewSubscriber(dstHost, dstStore, dstLnkSys, "/indexer/ingest/testnet") if err != nil { panic(err) } diff --git a/dagsync/subscriber.go b/dagsync/subscriber.go index 9746e5e..ec514be 100644 --- a/dagsync/subscriber.go +++ b/dagsync/subscriber.go @@ -133,9 +133,9 @@ type SyncFinished struct { PeerID peer.ID // Count is the number of CID synced. Count int - // AsyncErr is used to return a failure to asynchronous sync in response to - // an announcement. - AsyncErr error + // Err is used to return a failure to complete an asynchronous sync in + // response to an announcement. + Err error } // handler holds state that is specific to a peer @@ -862,9 +862,9 @@ func (h *handler) asyncSyncAdChain(ctx context.Context) { return } h.subscriber.inEvents <- SyncFinished{ - Cid: nextCid, - PeerID: h.peerID, - AsyncErr: err, + Cid: nextCid, + PeerID: h.peerID, + Err: err, } return } diff --git a/dagsync/subscriber_test.go b/dagsync/subscriber_test.go index 9e39caa..31d6a46 100644 --- a/dagsync/subscriber_test.go +++ b/dagsync/subscriber_test.go @@ -46,10 +46,10 @@ func TestScopedBlockHook(t *testing.T) { err := quick.Check(func(ll llBuilder) bool { return t.Run("Quickcheck", func(t *testing.T) { ds := dssync.MutexWrap(datastore.NewMapDatastore()) - pubHost := test.MkTestHost(t) + pubHost, privKey := test.MkTestHostPK(t) lsys := test.MkLinkSystem(ds) - pub, err := dtsync.NewPublisher(pubHost, ds, lsys, testTopic) + pub, err := ipnisync.NewPublisher(lsys, privKey, ipnisync.WithStreamHost(pubHost)) require.NoError(t, err) head := ll.Build(t, lsys) @@ -64,8 +64,6 @@ func TestScopedBlockHook(t *testing.T) { subDS := dssync.MutexWrap(datastore.NewMapDatastore()) subLsys := test.MkLinkSystem(subDS) - require.NoError(t, test.WaitForP2PPublisher(pub, subHost, testTopic)) - var calledGeneralBlockHookTimes int64 sub, err := dagsync.NewSubscriber(subHost, subDS, subLsys, testTopic, dagsync.BlockHook(func(i peer.ID, c cid.Cid, _ dagsync.SegmentSyncActions) { @@ -114,9 +112,9 @@ func TestSyncedCidsReturned(t *testing.T) { err := quick.Check(func(ll llBuilder) bool { return t.Run("Quickcheck", func(t *testing.T) { ds := dssync.MutexWrap(datastore.NewMapDatastore()) - pubHost := test.MkTestHost(t) + pubHost, privKey := test.MkTestHostPK(t) lsys := test.MkLinkSystem(ds) - pub, err := dtsync.NewPublisher(pubHost, ds, lsys, testTopic) + pub, err := ipnisync.NewPublisher(lsys, privKey, ipnisync.WithStreamHost(pubHost)) require.NoError(t, err) head := ll.Build(t, lsys) @@ -131,8 +129,6 @@ func TestSyncedCidsReturned(t *testing.T) { subDS := dssync.MutexWrap(datastore.NewMapDatastore()) subLsys := test.MkLinkSystem(subDS) - require.NoError(t, test.WaitForP2PPublisher(pub, subHost, testTopic)) - sub, err := dagsync.NewSubscriber(subHost, subDS, subLsys, testTopic, dagsync.StrictAdsSelector(false)) require.NoError(t, err) @@ -175,12 +171,11 @@ func TestConcurrentSync(t *testing.T) { for i := 0; i < publisherCount; i++ { ds := dssync.MutexWrap(datastore.NewMapDatastore()) - pubHost := test.MkTestHost(t) + pubHost, privKey := test.MkTestHostPK(t) lsys := test.MkLinkSystem(ds) - pub, err := dtsync.NewPublisher(pubHost, ds, lsys, testTopic) + pub, err := ipnisync.NewPublisher(lsys, privKey, ipnisync.WithStreamHost(pubHost)) require.NoError(t, err) - require.NoError(t, test.WaitForP2PPublisher(pub, subHost, testTopic)) publishers = append(publishers, pubMeta{pub, pubHost}) @@ -391,11 +386,11 @@ func TestRoundTripSimple(t *testing.T) { func TestRoundTrip(t *testing.T) { // Init dagsync publisher and subscriber srcStore1 := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost1 := test.MkTestHost(t) + srcHost1, privKey1 := test.MkTestHostPK(t) srcLnkS1 := test.MkLinkSystem(srcStore1) srcStore2 := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost2 := test.MkTestHost(t) + srcHost2, privKey2 := test.MkTestHostPK(t) srcLnkS2 := test.MkLinkSystem(srcStore2) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) @@ -407,14 +402,14 @@ func TestRoundTrip(t *testing.T) { p2pSender1, err := p2psender.New(nil, "", p2psender.WithTopic(topics[0])) require.NoError(t, err) - pub1, err := dtsync.NewPublisher(srcHost1, srcStore1, srcLnkS1, "") + pub1, err := ipnisync.NewPublisher(srcLnkS1, privKey1, ipnisync.WithStreamHost(srcHost1)) require.NoError(t, err) defer pub1.Close() p2pSender2, err := p2psender.New(nil, "", p2psender.WithTopic(topics[1])) require.NoError(t, err) - pub2, err := dtsync.NewPublisher(srcHost2, srcStore2, srcLnkS2, "") + pub2, err := ipnisync.NewPublisher(srcLnkS2, privKey2, ipnisync.WithStreamHost(srcHost2)) require.NoError(t, err) defer pub2.Close() @@ -605,16 +600,16 @@ func TestSyncFinishedAlwaysDelivered(t *testing.T) { func TestMaxAsyncSyncs(t *testing.T) { // Create two publishers srcStore1 := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost1 := test.MkTestHost(t) + srcHost1, privKey1 := test.MkTestHostPK(t) srcLnkS1 := test.MkLinkSystem(srcStore1) - pub1, err := dtsync.NewPublisher(srcHost1, srcStore1, srcLnkS1, "") + pub1, err := ipnisync.NewPublisher(srcLnkS1, privKey1, ipnisync.WithStreamHost(srcHost1)) require.NoError(t, err) defer pub1.Close() srcStore2 := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost2 := test.MkTestHost(t) + srcHost2, privKey2 := test.MkTestHostPK(t) srcLnkS2 := test.MkLinkSystem(srcStore2) - pub2, err := dtsync.NewPublisher(srcHost2, srcStore2, srcLnkS2, "") + pub2, err := ipnisync.NewPublisher(srcLnkS2, privKey2, ipnisync.WithStreamHost(srcHost2)) require.NoError(t, err) defer pub2.Close() @@ -851,6 +846,7 @@ func TestIdleHandlerCleaner(t *testing.T) { type dagsyncPubSubBuilder struct { IsHttp bool + IsDtSync bool P2PAnnounce bool } @@ -886,11 +882,13 @@ func (b dagsyncPubSubBuilder) Build(t *testing.T, topicName string, pubSys hostS if b.IsHttp { pub, err = ipnisync.NewPublisher(pubSys.lsys, pubSys.privKey, ipnisync.WithHeadTopic(topicName), ipnisync.WithHTTPListenAddrs("127.0.0.1:0")) require.NoError(t, err) - require.NoError(t, test.WaitForHttpPublisher(pub)) - } else { + } else if b.IsDtSync { pub, err = dtsync.NewPublisher(pubSys.host, pubSys.ds, pubSys.lsys, topicName) require.NoError(t, err) require.NoError(t, test.WaitForP2PPublisher(pub, subSys.host, topicName)) + } else { + pub, err = ipnisync.NewPublisher(pubSys.lsys, pubSys.privKey, ipnisync.WithStreamHost(pubSys.host), ipnisync.WithHeadTopic(topicName)) + require.NoError(t, err) } subOpts = append(subOpts, dagsync.StrictAdsSelector(false)) diff --git a/dagsync/sync_test.go b/dagsync/sync_test.go index 6666bd3..00c7ffa 100644 --- a/dagsync/sync_test.go +++ b/dagsync/sync_test.go @@ -16,6 +16,7 @@ import ( "github.com/ipni/go-libipni/announce/p2psender" "github.com/ipni/go-libipni/dagsync" "github.com/ipni/go-libipni/dagsync/dtsync" + "github.com/ipni/go-libipni/dagsync/ipnisync" "github.com/ipni/go-libipni/dagsync/test" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" @@ -24,7 +25,7 @@ import ( func TestLatestSyncSuccess(t *testing.T) { srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost := test.MkTestHost(t) + srcHost, srcPrivKey := test.MkTestHostPK(t) srcLnkS := test.MkLinkSystem(srcStore) dstHost := test.MkTestHost(t) @@ -37,7 +38,7 @@ func TestLatestSyncSuccess(t *testing.T) { p2pSender, err := p2psender.New(nil, "", p2psender.WithTopic(topics[0])) require.NoError(t, err) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost)) require.NoError(t, err) defer pub.Close() @@ -47,8 +48,6 @@ func TestLatestSyncSuccess(t *testing.T) { require.NoError(t, err) defer sub.Close() - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, topics[0].String())) - watcher, cncl := sub.OnSyncFinished() defer cncl() @@ -67,7 +66,7 @@ func TestSyncFn(t *testing.T) { t.Parallel() srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost := test.MkTestHost(t) + srcHost, srcPrivKey := test.MkTestHostPK(t) srcLnkS := test.MkLinkSystem(srcStore) dstHost := test.MkTestHost(t) @@ -80,7 +79,7 @@ func TestSyncFn(t *testing.T) { p2pSender, err := p2psender.New(nil, "", p2psender.WithTopic(topics[0])) require.NoError(t, err) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost)) require.NoError(t, err) defer pub.Close() @@ -104,8 +103,6 @@ func TestSyncFn(t *testing.T) { // Store the whole chain in source node chainLnks := test.MkChain(srcLnkS, true) - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, topics[0].String())) - watcher, cancelWatcher := sub.OnSyncFinished() defer cancelWatcher() @@ -184,7 +181,7 @@ func TestPartialSync(t *testing.T) { srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) testStore := dssync.MutexWrap(datastore.NewMapDatastore()) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost := test.MkTestHost(t) + srcHost, srcPrivKey := test.MkTestHostPK(t) srcLnkS := test.MkLinkSystem(srcStore) testLnkS := test.MkLinkSystem(testStore) @@ -200,7 +197,7 @@ func TestPartialSync(t *testing.T) { p2pSender, err := p2psender.New(nil, "", p2psender.WithTopic(topics[0])) require.NoError(t, err) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost)) require.NoError(t, err) defer pub.Close() test.MkChain(srcLnkS, true) @@ -217,8 +214,6 @@ func TestPartialSync(t *testing.T) { err = srcHost.Connect(context.Background(), dstHost.Peerstore().PeerInfo(dstHost.ID())) require.NoError(t, err) - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, topics[0].String())) - watcher, cncl := sub.OnSyncFinished() defer cncl() @@ -250,7 +245,7 @@ func TestStepByStepSync(t *testing.T) { dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) srcLnkS := test.MkLinkSystem(srcStore) - srcHost := test.MkTestHost(t) + srcHost, srcPrivKey := test.MkTestHostPK(t) dstHost := test.MkTestHost(t) topics := test.WaitForMeshWithMessage(t, testTopic, srcHost, dstHost) @@ -260,7 +255,7 @@ func TestStepByStepSync(t *testing.T) { p2pSender, err := p2psender.New(nil, "", p2psender.WithTopic(topics[0])) require.NoError(t, err) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost)) require.NoError(t, err) defer pub.Close() @@ -270,8 +265,6 @@ func TestStepByStepSync(t *testing.T) { require.NoError(t, err) defer sub.Close() - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, topics[0].String())) - watcher, cncl := sub.OnSyncFinished() defer cncl() @@ -293,9 +286,9 @@ func TestLatestSyncFailure(t *testing.T) { t.Parallel() srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost := test.MkTestHost(t) + srcHost, srcPrivKey := test.MkTestHostPK(t) srcLnkS := test.MkLinkSystem(srcStore) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost)) require.NoError(t, err) defer pub.Close() @@ -320,8 +313,6 @@ func TestLatestSyncFailure(t *testing.T) { err = sub.SetLatestSync(srcHost.ID(), chainLnks[3].(cidlink.Link).Cid) require.NoError(t, err) - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, testTopic)) - watcher, cncl := sub.OnSyncFinished() t.Log("Testing sync fail when the other end does not have the data") @@ -345,50 +336,125 @@ func TestLatestSyncFailure(t *testing.T) { require.NoError(t, err) } -func TestAnnounce(t *testing.T) { - srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) +func TestSyncOnAnnounceDataTransfer(t *testing.T) { dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) + dstHost := test.MkTestHost(t) + dstLnkS := test.MkLinkSystem(dstStore) + + sub, err := dagsync.NewSubscriber(dstHost, dstStore, dstLnkS, testTopic, + dagsync.RecvAnnounce(), dagsync.StrictAdsSelector(false)) + require.NoError(t, err) + defer sub.Close() + + watcher, cncl := sub.OnSyncFinished() + defer cncl() + srcHost := test.MkTestHost(t) + srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) srcLnkS := test.MkLinkSystem(srcStore) - dstHost := test.MkTestHost(t) + pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + require.NoError(t, err) + defer pub.Close() + require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, testTopic)) srcHost.Peerstore().AddAddrs(dstHost.ID(), dstHost.Addrs(), time.Hour) dstHost.Peerstore().AddAddrs(srcHost.ID(), srcHost.Addrs(), time.Hour) + + // Store the whole chain in source node + chainLnks := test.MkChain(srcLnkS, true) + + t.Log("Testing announce-sync with dtsync publisher at:", pub.Addrs()) + pubInfo := peer.AddrInfo{ + ID: pub.ID(), + Addrs: pub.Addrs(), + } + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[2], chainLnks[2].(cidlink.Link).Cid) + require.NoError(t, err) + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[1], chainLnks[1].(cidlink.Link).Cid) + require.NoError(t, err) + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[0], chainLnks[0].(cidlink.Link).Cid) + require.NoError(t, err) +} + +func TestSyncOnAnnounceIPNI(t *testing.T) { + dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) + dstHost := test.MkTestHost(t) dstLnkS := test.MkLinkSystem(dstStore) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + sub, err := dagsync.NewSubscriber(dstHost, dstStore, dstLnkS, testTopic, + dagsync.RecvAnnounce(), dagsync.StrictAdsSelector(false)) + require.NoError(t, err) + defer sub.Close() + + watcher, cncl := sub.OnSyncFinished() + defer cncl() + + srcHost, srcPrivKey := test.MkTestHostPK(t) + srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) + srcLnkS := test.MkLinkSystem(srcStore) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost), ipnisync.WithHeadTopic(testTopic)) require.NoError(t, err) defer pub.Close() + srcHost.Peerstore().AddAddrs(dstHost.ID(), dstHost.Addrs(), time.Hour) + dstHost.Peerstore().AddAddrs(srcHost.ID(), srcHost.Addrs(), time.Hour) + + // Store the whole chain in source node + chainLnks := test.MkChain(srcLnkS, true) + + t.Log("Testing announce-sync with libp2phttp publisher at:", pub.Addrs()) + pubInfo := peer.AddrInfo{ + ID: pub.ID(), + Addrs: pub.Addrs(), + } + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[2], chainLnks[2].(cidlink.Link).Cid) + require.NoError(t, err) + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[1], chainLnks[1].(cidlink.Link).Cid) + require.NoError(t, err) + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[0], chainLnks[0].(cidlink.Link).Cid) + require.NoError(t, err) +} + +func TestSyncOnAnnounceHTTP(t *testing.T) { + dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) + dstHost := test.MkTestHost(t) + dstLnkS := test.MkLinkSystem(dstStore) + sub, err := dagsync.NewSubscriber(dstHost, dstStore, dstLnkS, testTopic, dagsync.RecvAnnounce(), dagsync.StrictAdsSelector(false)) require.NoError(t, err) defer sub.Close() - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, testTopic)) - watcher, cncl := sub.OnSyncFinished() defer cncl() + _, srcPrivKey := test.MkTestHostPK(t) + srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) + srcLnkS := test.MkLinkSystem(srcStore) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithHTTPListenAddrs("http://127.0.0.1:0"), ipnisync.WithHeadTopic(testTopic)) + require.NoError(t, err) + defer pub.Close() + // Store the whole chain in source node chainLnks := test.MkChain(srcLnkS, true) - srcHostInfo := peer.AddrInfo{ - ID: srcHost.ID(), - Addrs: srcHost.Addrs(), + t.Log("Testing announce-sync with HTTP publisher at:", pub.Addrs()) + pubInfo := peer.AddrInfo{ + ID: pub.ID(), + Addrs: pub.Addrs(), } - err = newAnnounceTest(pub, sub, dstStore, watcher, srcHostInfo, chainLnks[2], chainLnks[2].(cidlink.Link).Cid) + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[2], chainLnks[2].(cidlink.Link).Cid) require.NoError(t, err) - err = newAnnounceTest(pub, sub, dstStore, watcher, srcHostInfo, chainLnks[1], chainLnks[1].(cidlink.Link).Cid) + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[1], chainLnks[1].(cidlink.Link).Cid) require.NoError(t, err) - err = newAnnounceTest(pub, sub, dstStore, watcher, srcHostInfo, chainLnks[0], chainLnks[0].(cidlink.Link).Cid) + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[0], chainLnks[0].(cidlink.Link).Cid) require.NoError(t, err) } func TestCancelDeadlock(t *testing.T) { srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost := test.MkTestHost(t) + srcHost, srcPrivKey := test.MkTestHostPK(t) srcLnkS := test.MkLinkSystem(srcStore) dstHost := test.MkTestHost(t) @@ -396,7 +462,7 @@ func TestCancelDeadlock(t *testing.T) { dstHost.Peerstore().AddAddrs(srcHost.ID(), srcHost.Addrs(), time.Hour) dstLnkS := test.MkLinkSystem(dstStore) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost)) require.NoError(t, err) defer pub.Close() @@ -404,8 +470,6 @@ func TestCancelDeadlock(t *testing.T) { require.NoError(t, err) defer sub.Close() - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, testTopic)) - watcher, cncl := sub.OnSyncFinished() // Store the whole chain in source node diff --git a/dagsync/test/util.go b/dagsync/test/util.go index f9d625c..154f972 100644 --- a/dagsync/test/util.go +++ b/dagsync/test/util.go @@ -3,6 +3,7 @@ package test import ( "bytes" "context" + "crypto/rand" "errors" "fmt" "io" @@ -18,13 +19,14 @@ import ( "github.com/ipld/go-ipld-prime/fluent" cidlink "github.com/ipld/go-ipld-prime/linking/cid" basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/ipni/go-libipni/dagsync/dtsync/head" "github.com/ipni/go-libipni/dagsync/ipnisync" - "github.com/ipni/go-libipni/dagsync/p2p/protocol/head" "github.com/ipni/go-libipni/ingest/schema" "github.com/ipni/go-libipni/maurl" "github.com/ipni/go-libipni/test" "github.com/libp2p/go-libp2p" pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" @@ -237,6 +239,12 @@ func MkTestHost(t *testing.T, options ...libp2p.Option) host.Host { return h } +func MkTestHostPK(t *testing.T) (host.Host, crypto.PrivKey) { + privKey, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + return MkTestHost(t, libp2p.Identity(privKey)), privKey +} + // Return the chain with all nodes or just half of it for testing func MkChain(lsys ipld.LinkSystem, full bool) []ipld.Link { out := make([]ipld.Link, 4) From f5acb82505cbe3123a284d2d82e8e1b762c0c955 Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 25 Aug 2023 16:47:04 -0700 Subject: [PATCH 22/29] update comments --- announce/httpsender/sender.go | 8 ++++---- dagsync/ipnisync/sync.go | 13 +++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/announce/httpsender/sender.go b/announce/httpsender/sender.go index 14663ed..0834006 100644 --- a/announce/httpsender/sender.go +++ b/announce/httpsender/sender.go @@ -27,10 +27,10 @@ type Sender struct { userAgent string } -// New creates a new Sender that sends announce messages over HTTP. Announce -// messages are sent to the specified URLs. The addresses in announce messages -// are modified to include the specified peerID, which is necessary to -// communicate the publisher ID over HTTP. +// New creates a new Sender that sends advertisement announcement messages over +// HTTP. Announcements are sent directly to the specified URLs. The specified +// peerID is added to the multiaddrs contained in the announcements, which is +// how the publisher ID is communicated over HTTP. func New(announceURLs []*url.URL, peerID peer.ID, options ...Option) (*Sender, error) { if len(announceURLs) == 0 { return nil, errors.New("no announce urls") diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index e71a89f..8be1d3e 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -267,12 +267,6 @@ func (s *Syncer) walkFetch(ctx context.Context, rootCid cid.Cid, sel selector.Se return r, err } - // get the direct node. - rootNode, err := getMissingLs.Load(ipld.LinkContext{Ctx: ctx}, cidlink.Link{Cid: rootCid}, basicnode.Prototype.Any) - if err != nil { - return nil, fmt.Errorf("failed to load node for root cid %s: %w", rootCid, err) - } - progress := traversal.Progress{ Cfg: &traversal.Config{ Ctx: ctx, @@ -281,6 +275,13 @@ func (s *Syncer) walkFetch(ctx context.Context, rootCid cid.Cid, sel selector.Se }, Path: datamodel.NewPath([]datamodel.PathSegment{}), } + + // get the direct node. + rootNode, err := getMissingLs.Load(ipld.LinkContext{Ctx: ctx}, cidlink.Link{Cid: rootCid}, basicnode.Prototype.Any) + if err != nil { + return nil, fmt.Errorf("failed to load node for root cid %s: %w", rootCid, err) + } + err = progress.WalkMatching(rootNode, sel, func(_ traversal.Progress, _ datamodel.Node) error { return nil }) if err != nil { return nil, err From 2194d3a101983872578fb9c7fe7afb6395c07c9d Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 25 Aug 2023 17:44:28 -0700 Subject: [PATCH 23/29] Update libp2phttp --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 27f9061..fff076c 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/ipfs/go-ipld-format v0.3.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-ipld-prime v0.20.0 - github.com/libp2p/go-libp2p v0.29.1-0.20230824185545-1af5d3baeec0 + github.com/libp2p/go-libp2p v0.29.1-0.20230825222710-ed42d2fe6cee github.com/libp2p/go-libp2p-gostream v0.6.0 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-msgio v0.3.0 diff --git a/go.sum b/go.sum index 1d77e3e..6f49c46 100644 --- a/go.sum +++ b/go.sum @@ -305,8 +305,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.29.1-0.20230824185545-1af5d3baeec0 h1:qaUGo0lQDfzTCsdnPFDGCNbYAq9tJvhUL+c/WzPgetg= -github.com/libp2p/go-libp2p v0.29.1-0.20230824185545-1af5d3baeec0/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= +github.com/libp2p/go-libp2p v0.29.1-0.20230825222710-ed42d2fe6cee h1:UioCkyX/qjEs5mzVoBfZUChXtyhtXG9salRwQ8iUDfI= +github.com/libp2p/go-libp2p v0.29.1-0.20230825222710-ed42d2fe6cee/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU= From 6828f75e198d4a4de876af964a77ece43d2e309e Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 29 Aug 2023 07:46:42 -0700 Subject: [PATCH 24/29] Update libp2p --- go.mod | 33 +++++++++++++++--------------- go.sum | 63 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 47 insertions(+), 49 deletions(-) diff --git a/go.mod b/go.mod index fff076c..a1318d3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ipni/go-libipni -go 1.19 +go 1.20 require ( github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7 @@ -13,19 +13,19 @@ require ( github.com/ipfs/go-ipld-format v0.3.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-ipld-prime v0.20.0 - github.com/libp2p/go-libp2p v0.29.1-0.20230825222710-ed42d2fe6cee + github.com/libp2p/go-libp2p v0.30.1-0.20230829055202-ac038dbf2d38 github.com/libp2p/go-libp2p-gostream v0.6.0 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-msgio v0.3.0 github.com/mr-tron/base58 v1.2.0 - github.com/multiformats/go-multiaddr v0.10.1 + github.com/multiformats/go-multiaddr v0.11.0 github.com/multiformats/go-multicodec v0.9.0 github.com/multiformats/go-multihash v0.2.3 github.com/multiformats/go-multistream v0.4.1 github.com/multiformats/go-varint v0.0.7 github.com/stretchr/testify v1.8.4 github.com/whyrusleeping/cbor-gen v0.0.0-20230418232409-daab9ece03a0 - golang.org/x/crypto v0.11.0 + golang.org/x/crypto v0.12.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 google.golang.org/protobuf v1.30.0 ) @@ -58,14 +58,14 @@ require ( github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect + github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c // indirect github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect github.com/huin/goupnp v1.2.0 // indirect github.com/ipfs/go-block-format v0.0.3 // indirect github.com/ipfs/go-ipfs-pq v0.0.2 // indirect @@ -87,7 +87,7 @@ require ( github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect github.com/libp2p/go-nat v0.2.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect - github.com/libp2p/go-reuseport v0.3.0 // indirect + github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.1 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-isatty v0.0.19 // indirect @@ -102,7 +102,7 @@ require ( github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/onsi/ginkgo/v2 v2.11.0 // indirect - github.com/opencontainers/runtime-spec v1.0.2 // indirect + github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -113,9 +113,8 @@ require ( github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/quic-go/qpack v0.4.0 // indirect - github.com/quic-go/qtls-go1-19 v0.3.3 // indirect - github.com/quic-go/qtls-go1-20 v0.2.3 // indirect - github.com/quic-go/quic-go v0.36.4 // indirect + github.com/quic-go/qtls-go1-20 v0.3.3 // indirect + github.com/quic-go/quic-go v0.38.1 // indirect github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect @@ -129,14 +128,14 @@ require ( go.uber.org/dig v1.17.0 // indirect go.uber.org/fx v1.20.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.24.0 // indirect - golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect + go.uber.org/zap v1.25.0 // indirect + golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.12.0 // indirect + golang.org/x/net v0.14.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect - golang.org/x/tools v0.11.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect + golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/go.sum b/go.sum index 6f49c46..f60bfb4 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+uqi7GyzaLa5MH7qlSLBZtRdiA= -github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= +github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo= +github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -193,8 +193,8 @@ github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= -github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= +github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -305,8 +305,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.29.1-0.20230825222710-ed42d2fe6cee h1:UioCkyX/qjEs5mzVoBfZUChXtyhtXG9salRwQ8iUDfI= -github.com/libp2p/go-libp2p v0.29.1-0.20230825222710-ed42d2fe6cee/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= +github.com/libp2p/go-libp2p v0.30.1-0.20230829055202-ac038dbf2d38 h1:rwTWCb3A7Xv7BEB5JGJBjejaTVXyGm2Mu/iBvhYPQPE= +github.com/libp2p/go-libp2p v0.30.1-0.20230829055202-ac038dbf2d38/go.mod h1:W/FEK1c/t04PbRH3fA9i5oucu5YcgrG0JVoBWT1B7Eg= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU= @@ -320,8 +320,8 @@ github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= github.com/libp2p/go-nat v0.2.0/go.mod h1:3MJr+GRpRkyT65EpVPBstXLvOlAPzUVlG6Pwg9ohLJk= github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= -github.com/libp2p/go-reuseport v0.3.0 h1:iiZslO5byUYZEg9iCwJGf5h+sf1Agmqx2V2FDjPyvUw= -github.com/libp2p/go-reuseport v0.3.0/go.mod h1:laea40AimhtfEqysZ71UpYj4S+R9VpH8PgqLo7L+SwI= +github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= +github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= github.com/libp2p/go-yamux/v4 v4.0.1 h1:FfDR4S1wj6Bw2Pqbc8Uz7pCxeRBPbwsBbEdfwiCypkQ= github.com/libp2p/go-yamux/v4 v4.0.1/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= @@ -365,8 +365,8 @@ github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9 github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= -github.com/multiformats/go-multiaddr v0.10.1 h1:HghtFrWyZEPrpTvgAMFJi6gFdgHfs2cb0pyfDsk+lqU= -github.com/multiformats/go-multiaddr v0.10.1/go.mod h1:jLEZsA61rwWNZQTHHnqq2HNa+4os/Hz54eqiRnsRqYQ= +github.com/multiformats/go-multiaddr v0.11.0 h1:XqGyJ8ufbCE0HmTDwx2kPdsrQ36AGPZNZX6s6xfJH10= +github.com/multiformats/go-multiaddr v0.11.0/go.mod h1:gWUm0QLR4thQ6+ZF6SXUw8YjtwQSPapICM+NmCkxHSM= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= @@ -396,8 +396,9 @@ github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= -github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= +github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= @@ -428,12 +429,10 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= -github.com/quic-go/qtls-go1-19 v0.3.3 h1:wznEHvJwd+2X3PqftRha0SUKmGsnb6dfArMhy9PeJVE= -github.com/quic-go/qtls-go1-19 v0.3.3/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= -github.com/quic-go/qtls-go1-20 v0.2.3 h1:m575dovXn1y2ATOb1XrRFcrv0F+EQmlowTkoraNkDPI= -github.com/quic-go/qtls-go1-20 v0.2.3/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= -github.com/quic-go/quic-go v0.36.4 h1:CXn/ZLN5Vntlk53fjR+kUMC8Jt7flfQe+I5Ty5A+k0o= -github.com/quic-go/quic-go v0.36.4/go.mod h1:qxQumdeKw5GmWs1OsTZZnOxzSI+RJWuhf1O8FN35L2o= +github.com/quic-go/qtls-go1-20 v0.3.3 h1:17/glZSLI9P9fDAeyCHBFSWSqJcwx1byhLwP5eUIDCM= +github.com/quic-go/qtls-go1-20 v0.3.3/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= +github.com/quic-go/quic-go v0.38.1 h1:M36YWA5dEhEeT+slOu/SwMEucbYd0YFidxG3KlGPZaE= +github.com/quic-go/quic-go v0.38.1/go.mod h1:ijnZM7JsFIkp4cRyjxJNIzdSfCLmUMg9wdyhGmg+SN4= github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU= github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= @@ -535,7 +534,7 @@ go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= go.uber.org/fx v1.20.0 h1:ZMC/pnRvhsthOZh9MZjMq5U8Or3mA9zBSPaLnzs3ihQ= go.uber.org/fx v1.20.0/go.mod h1:qCUj0btiR3/JnanEr1TYEePfSw6o/4qYJscgvzQ5Ub0= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -546,8 +545,8 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20200411211856-f5505b9728dd h1:BNJlw5kRTzdmyfh5U8F93HA2OwkP7ZGwA51eJ/0wKOU= go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= @@ -563,8 +562,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -573,8 +572,8 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= +golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -621,8 +620,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -671,16 +670,16 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -719,8 +718,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= -golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= +golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E= +golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 1ee93ea76eb6cfcb55158b988574bd401236a6dd Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 29 Aug 2023 08:40:52 -0700 Subject: [PATCH 25/29] gostream relocated --- dagsync/dtsync/head/head.go | 6 +++--- go.mod | 1 - go.sum | 2 -- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/dagsync/dtsync/head/head.go b/dagsync/dtsync/head/head.go index d39690a..f6ec73e 100644 --- a/dagsync/dtsync/head/head.go +++ b/dagsync/dtsync/head/head.go @@ -13,11 +13,11 @@ import ( "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" - gostream "github.com/libp2p/go-libp2p-gostream" "github.com/libp2p/go-libp2p/core/host" - peer "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" - multistream "github.com/multiformats/go-multistream" + "github.com/libp2p/go-libp2p/p2p/net/gostream" + "github.com/multiformats/go-multistream" ) const closeTimeout = 30 * time.Second diff --git a/go.mod b/go.mod index a1318d3..394fea1 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-ipld-prime v0.20.0 github.com/libp2p/go-libp2p v0.30.1-0.20230829055202-ac038dbf2d38 - github.com/libp2p/go-libp2p-gostream v0.6.0 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-msgio v0.3.0 github.com/mr-tron/base58 v1.2.0 diff --git a/go.sum b/go.sum index f60bfb4..be955d0 100644 --- a/go.sum +++ b/go.sum @@ -309,8 +309,6 @@ github.com/libp2p/go-libp2p v0.30.1-0.20230829055202-ac038dbf2d38 h1:rwTWCb3A7Xv github.com/libp2p/go-libp2p v0.30.1-0.20230829055202-ac038dbf2d38/go.mod h1:W/FEK1c/t04PbRH3fA9i5oucu5YcgrG0JVoBWT1B7Eg= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= -github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU= -github.com/libp2p/go-libp2p-gostream v0.6.0/go.mod h1:Nywu0gYZwfj7Jc91PQvbGU8dIpqbQQkjWgDuOrFaRdA= github.com/libp2p/go-libp2p-pubsub v0.9.3 h1:ihcz9oIBMaCK9kcx+yHWm3mLAFBMAUsM4ux42aikDxo= github.com/libp2p/go-libp2p-pubsub v0.9.3/go.mod h1:RYA7aM9jIic5VV47WXu4GkcRxRhrdElWf8xtyli+Dzc= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= From 0c9c4cae11e0b5ce75b5eb0fa31ae84b38dd8258 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 29 Aug 2023 09:18:14 -0700 Subject: [PATCH 26/29] update graphsync --- go.mod | 22 +++++++++--------- go.sum | 72 +++++++++++++++++++++++++++------------------------------- 2 files changed, 45 insertions(+), 49 deletions(-) diff --git a/go.mod b/go.mod index 394fea1..685a639 100644 --- a/go.mod +++ b/go.mod @@ -9,10 +9,10 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.4 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 - github.com/ipfs/go-graphsync v0.14.7 - github.com/ipfs/go-ipld-format v0.3.0 + github.com/ipfs/go-graphsync v0.14.8 + github.com/ipfs/go-ipld-format v0.4.0 github.com/ipfs/go-log/v2 v2.5.1 - github.com/ipld/go-ipld-prime v0.20.0 + github.com/ipld/go-ipld-prime v0.21.0 github.com/libp2p/go-libp2p v0.30.1-0.20230829055202-ac038dbf2d38 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-msgio v0.3.0 @@ -50,7 +50,7 @@ require ( github.com/francoispqt/gojay v1.2.13 // indirect github.com/gammazero/deque v0.2.1 // indirect github.com/go-logr/logr v1.2.4 // indirect - github.com/go-logr/stdr v1.2.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -66,13 +66,13 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect github.com/huin/goupnp v1.2.0 // indirect - github.com/ipfs/go-block-format v0.0.3 // indirect - github.com/ipfs/go-ipfs-pq v0.0.2 // indirect + github.com/ipfs/go-block-format v0.1.2 // indirect + github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect - github.com/ipfs/go-ipld-cbor v0.0.5 // indirect + github.com/ipfs/go-ipld-cbor v0.0.6 // indirect github.com/ipfs/go-log v1.0.5 // indirect - github.com/ipfs/go-peertaskqueue v0.8.0 // indirect - github.com/ipld/go-codec-dagpb v1.5.0 // indirect + github.com/ipfs/go-peertaskqueue v0.8.1 // indirect + github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect @@ -121,8 +121,8 @@ require ( github.com/smartystreets/assertions v1.13.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/urfave/cli/v2 v2.0.0 // indirect - go.opentelemetry.io/otel v1.3.0 // indirect - go.opentelemetry.io/otel/trace v1.3.0 // indirect + go.opentelemetry.io/otel v1.13.0 // indirect + go.opentelemetry.io/otel/trace v1.13.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.0 // indirect go.uber.org/fx v1.20.0 // indirect diff --git a/go.sum b/go.sum index be955d0..b70cec8 100644 --- a/go.sum +++ b/go.sum @@ -90,7 +90,7 @@ github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gammazero/channelqueue v0.2.1 h1:AcK6wnLrj8koTTn3RxjRCyfmS677TjhIZb1FSMi14qc= github.com/gammazero/channelqueue v0.2.1/go.mod h1:824o5HHE+yO1xokh36BIuSv8YWwXW0364ku91eRMFS4= @@ -102,12 +102,11 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclK github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.0 h1:j4LrlVXgrbIWO83mmQUnK0Hi+YnbD+vzrE1z/EphbFE= -github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= @@ -146,7 +145,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -201,15 +199,14 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= -github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/dDTpMc= -github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= -github.com/ipfs/go-blockservice v0.3.0 h1:cDgcZ+0P0Ih3sl8+qjFr2sVaMdysg/YZpLj5WJ8kiiw= +github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= +github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE= +github.com/ipfs/go-blockservice v0.5.0 h1:B2mwhhhVQl2ntW2EIpaWPwSCxSuqr5fFA93Ms4bYLEY= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10/go.mod h1:/BYOuUoxkE+0f6tGzlzMvycuN+5l35VOR4Bpg2sCmds= github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= @@ -219,30 +216,30 @@ github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0M github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= -github.com/ipfs/go-graphsync v0.14.7 h1:V90NORSdCpUHAgqQhApU/bmPSLOnwtSHM2v7R90k9Do= -github.com/ipfs/go-graphsync v0.14.7/go.mod h1:yT0AfjFgicOoWdAlUJ96tQ5AkuGI4r1taIQX/aHbBQo= +github.com/ipfs/go-graphsync v0.14.8 h1:NFFHquTNnwPi05tJhdpPj4CJMnqRBLxpZd+IfPRauf4= +github.com/ipfs/go-graphsync v0.14.8/go.mod h1:qyHjUvHey6EfKUDMQPwCuVkMOurRG3hcjRm+FaVP6bE= github.com/ipfs/go-ipfs-blockstore v1.2.0 h1:n3WTeJ4LdICWs/0VSfjHrlqpPpl6MZ+ySd3j8qz0ykw= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= -github.com/ipfs/go-ipfs-exchange-interface v0.1.0 h1:TiMekCrOGQuWYtZO3mf4YJXDIdNgnKWZ9IE3fGlnWfo= -github.com/ipfs/go-ipfs-exchange-offline v0.2.0 h1:2PF4o4A7W656rC0RxuhUace997FTcDTcIQ6NoEtyjAI= +github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck05pqUw4mWNW9Pw6Y= +github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= -github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY= -github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= +github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= +github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= -github.com/ipfs/go-ipld-cbor v0.0.5 h1:ovz4CHKogtG2KB/h1zUp5U0c/IzZrL435rCh5+K/5G8= -github.com/ipfs/go-ipld-cbor v0.0.5/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= +github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0= +github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= -github.com/ipfs/go-ipld-format v0.3.0 h1:Mwm2oRLzIuUwEPewWAWyMuuBQUsn3awfFEYVb8akMOQ= -github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= -github.com/ipfs/go-ipld-legacy v0.1.0 h1:wxkkc4k8cnvIGIjPO0waJCe7SHEyFgl+yQdafdjGrpA= -github.com/ipfs/go-libipfs v0.1.0 h1:I6CrHHp4cIiqsWJPVU3QBH4BZrRWSljS2aAbA3Eg9AY= +github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSghBlQ= +github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= +github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2cdcc= +github.com/ipfs/go-libipfs v0.6.0 h1:3FuckAJEm+zdHbHbf6lAyk0QUzc45LsFcGw102oBCZM= github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= @@ -251,17 +248,17 @@ github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBW github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= -github.com/ipfs/go-merkledag v0.8.1 h1:N3yrqSre/ffvdwtHL4MXy0n7XH+VzN8DlzDrJySPa94= +github.com/ipfs/go-merkledag v0.10.0 h1:IUQhj/kzTZfam4e+LnaEpoiZ9vZF6ldimVlby+6OXL4= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= -github.com/ipfs/go-peertaskqueue v0.8.0 h1:JyNO144tfu9bx6Hpo119zvbEL9iQ760FHOiJYsUjqaU= -github.com/ipfs/go-peertaskqueue v0.8.0/go.mod h1:cz8hEnnARq4Du5TGqiWKgMr/BOSQ5XOgMOh1K5YYKKM= -github.com/ipfs/go-unixfs v0.4.3 h1:EdDc1sNZNFDUlo4UrVAvvAofVI5EwTnKu8Nv8mgXkWQ= -github.com/ipfs/go-unixfsnode v1.5.2 h1:CvsiTt58W2uR5dD8bqQv+aAY0c1qolmXmSyNbPHYiew= -github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= -github.com/ipld/go-codec-dagpb v1.5.0 h1:RspDRdsJpLfgCI0ONhTAnbHdySGD4t+LHSPK4X1+R0k= -github.com/ipld/go-codec-dagpb v1.5.0/go.mod h1:0yRIutEFD8o1DGVqw4RSHh+BUTlJA9XWldxaaWR/o4g= -github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= -github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= +github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= +github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= +github.com/ipfs/go-unixfs v0.4.4 h1:D/dLBOJgny5ZLIur2vIXVQVW0EyDHdOMBDEhgHrt6rY= +github.com/ipfs/go-unixfsnode v1.7.4 h1:iLvKyAVKUYOIAW2t4kDYqsT7VLGj31eXJE2aeqGfbwA= +github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= +github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= +github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= +github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E= +github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -380,7 +377,6 @@ github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= @@ -496,7 +492,7 @@ github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg= github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U= +github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= @@ -516,11 +512,11 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/otel v1.3.0 h1:APxLf0eiBwLl+SOXiJJCVYzA1OOJNyAoV8C5RNRyy7Y= -go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= +go.opentelemetry.io/otel v1.13.0 h1:1ZAKnNQKwBBxFtww/GwxNUyTf0AxkZzrukO8MeXqe4Y= +go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= go.opentelemetry.io/otel/sdk v1.2.0 h1:wKN260u4DesJYhyjxDa7LRFkuhH7ncEVKU37LWcyNIo= -go.opentelemetry.io/otel/trace v1.3.0 h1:doy8Hzb1RJ+I3yFhtDmwNc7tIyw1tNMOIsyPzp1NOGY= -go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= +go.opentelemetry.io/otel/trace v1.13.0 h1:CBgRZ6ntv+Amuj1jDsMhZtlAPT6gbyIRdaIzFhfBSdY= +go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= From c4fda5e3848758ab1d2bd98efec835d63781876e Mon Sep 17 00:00:00 2001 From: gammazero Date: Wed, 30 Aug 2023 09:48:56 -0700 Subject: [PATCH 27/29] Update to release version of libp2p --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 685a639..e11f16c 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/ipfs/go-ipld-format v0.4.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-ipld-prime v0.21.0 - github.com/libp2p/go-libp2p v0.30.1-0.20230829055202-ac038dbf2d38 + github.com/libp2p/go-libp2p v0.31.0 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-msgio v0.3.0 github.com/mr-tron/base58 v1.2.0 diff --git a/go.sum b/go.sum index b70cec8..1ea0ce7 100644 --- a/go.sum +++ b/go.sum @@ -302,8 +302,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.30.1-0.20230829055202-ac038dbf2d38 h1:rwTWCb3A7Xv7BEB5JGJBjejaTVXyGm2Mu/iBvhYPQPE= -github.com/libp2p/go-libp2p v0.30.1-0.20230829055202-ac038dbf2d38/go.mod h1:W/FEK1c/t04PbRH3fA9i5oucu5YcgrG0JVoBWT1B7Eg= +github.com/libp2p/go-libp2p v0.31.0 h1:LFShhP8F6xthWiBBq3euxbKjZsoRajVEyBS9snfHxYg= +github.com/libp2p/go-libp2p v0.31.0/go.mod h1:W/FEK1c/t04PbRH3fA9i5oucu5YcgrG0JVoBWT1B7Eg= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-pubsub v0.9.3 h1:ihcz9oIBMaCK9kcx+yHWm3mLAFBMAUsM4ux42aikDxo= From b2617d498154897830b73bbab93f4b5594c578b4 Mon Sep 17 00:00:00 2001 From: gammazero Date: Wed, 30 Aug 2023 10:34:57 -0700 Subject: [PATCH 28/29] update comment --- dagsync/ipnisync/option.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dagsync/ipnisync/option.go b/dagsync/ipnisync/option.go index dcc6be8..6ba59f2 100644 --- a/dagsync/ipnisync/option.go +++ b/dagsync/ipnisync/option.go @@ -82,8 +82,8 @@ func WithStartServer(start bool) Option { } } -// WithRequireTLS tells whether to allow the publisher to serve non-secure http -// (false) or to require https (true). Default is false, allowing non-secure +// WithRequireTLS tells whether to allow the publisher to require https (true) +// or to serve non-secure http (false). Default is false, allowing non-secure // HTTP. func WithRequireTLS(require bool) Option { return func(c *config) error { @@ -123,8 +123,8 @@ func getClientOpts(opts []ClientOption) clientConfig { return cfg } -// ClientAuthServerPeerID tells the sync client that we MUST -// authenticate the Server's PeerID. +// ClientAuthServerPeerID tells the sync client that it must authenticate the +// server's peer ID. func ClientAuthServerPeerID(require bool) ClientOption { return func(c *clientConfig) { c.authPeerID = require From e6d63d9a85c7bfa683b117354ffb53b367c5d17c Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 31 Aug 2023 13:39:16 -0700 Subject: [PATCH 29/29] Use IPNIPath for libp2p protocol ID --- dagsync/ipnisync/ipnipath.go | 16 ++++++++++++---- dagsync/ipnisync/sync.go | 3 --- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/dagsync/ipnisync/ipnipath.go b/dagsync/ipnisync/ipnipath.go index 41077ca..650fcf4 100644 --- a/dagsync/ipnisync/ipnipath.go +++ b/dagsync/ipnisync/ipnipath.go @@ -1,7 +1,15 @@ package ipnisync -import "path" +import "github.com/libp2p/go-libp2p/core/protocol" -const protoVersion = "v1" - -var IPNIPath = path.Join("/ipni", protoVersion, "ad") +const ( + // IPNIPath is the path that the Publisher expects as the last port of the + // HTTP request URL path. The sync client automatically adds this to the + // request path. + IPNIPath = "/ipni/v1/ad" + // ProtocolID is the libp2p protocol ID used to identify the ipni-sync + // protocol. With libp2phttp this protocol ID maps directly to a single + // HTTP path, so the value of the protocol ID is the same as the IPNI path + // for the ipni-sync protocol. + ProtocolID = protocol.ID(IPNIPath) +) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index 8be1d3e..b400944 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -25,13 +25,10 @@ import ( "github.com/ipni/go-libipni/maurl" "github.com/ipni/go-libipni/mautil" "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/protocol" libp2phttp "github.com/libp2p/go-libp2p/p2p/http" "github.com/multiformats/go-multihash" ) -const ProtocolID = protocol.ID("/ipnisync/v1") - var log = logging.Logger("dagsync/ipnisync") var ErrNoHTTPServer = errors.New("publisher has libp2p server without HTTP")