Skip to content

Commit

Permalink
Upgrade libp2p (enables go 1.21 support) (erigontech#8288)
Browse files Browse the repository at this point in the history
Closes erigontech#8078 

This change is primarily intended to support go 1.21, but as a
side-effect requires updating libp2p, which in turn triggers an update
of golang.org/x/exp which creates quite a bit of (simple) churn in the
slice sorting.

This change introduces a new `cmp.Compare` function which can be used to
return an integer satisfying the compare interface for slice sorting.

In order to continue to support mplex for libp2p, the change references
github.com/libp2p/go-libp2p-mplex instead. Please see the PR at
libp2p/go-libp2p#2498 for the official usptream
comment that indicates official support for mplex being moved to this
location.

Co-authored-by: Jason Yellick <jason@enya.ai>
  • Loading branch information
2 people authored and lemenkov committed Oct 2, 2023
1 parent c91efdc commit 408dc14
Show file tree
Hide file tree
Showing 20 changed files with 118 additions and 97 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: '1.19'
go-version: '1.20'
- name: Install dependencies on Linux
if: runner.os == 'Linux'
run: sudo apt update && sudo apt install build-essential
Expand Down Expand Up @@ -84,7 +84,7 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: '1.19'
go-version: '1.20'

- uses: actions/cache@v3
with:
Expand Down
2 changes: 1 addition & 1 deletion cmd/sentinel/sentinel/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ import (
"github.com/ledgerwatch/erigon/cl/clparams"
"github.com/ledgerwatch/log/v3"
"github.com/libp2p/go-libp2p"
mplex "github.com/libp2p/go-libp2p-mplex"
"github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/p2p/muxer/mplex"
"github.com/libp2p/go-libp2p/p2p/security/noise"
"github.com/libp2p/go-libp2p/p2p/transport/tcp"
"github.com/multiformats/go-multiaddr"
Expand Down
3 changes: 2 additions & 1 deletion core/blockchain.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (

"github.com/ledgerwatch/erigon-lib/chain"
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/cmp"
"github.com/ledgerwatch/erigon-lib/common/fixedgas"

"github.com/ledgerwatch/erigon/common/math"
Expand Down Expand Up @@ -184,7 +185,7 @@ func ExecuteBlockEphemerally(

stateSyncReceipt := &types.Receipt{}
if chainConfig.Consensus == chain.BorConsensus && len(blockLogs) > 0 {
slices.SortStableFunc(blockLogs, func(i, j *types.Log) bool { return i.Index < j.Index })
slices.SortStableFunc(blockLogs, func(i, j *types.Log) int { return cmp.Compare(i.Index, j.Index) })

if len(blockLogs) > len(logs) {
stateSyncReceipt.Logs = blockLogs[len(logs):] // get state-sync logs from `state.Logs()`
Expand Down
11 changes: 11 additions & 0 deletions erigon-lib/common/cmp/cmp.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,14 @@ func Max[T constraints.Ordered](a, b T) T {
}
return b
}

func Compare[T constraints.Ordered](a, b T) int {
switch {
case a < b:
return -1
case a == b:
return 0
default:
return 1
}
}
29 changes: 17 additions & 12 deletions erigon-lib/compress/compress.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (

"github.com/c2h5oh/datasize"
"github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/cmp"
dir2 "github.com/ledgerwatch/erigon-lib/common/dir"
"github.com/ledgerwatch/erigon-lib/etl"
"github.com/ledgerwatch/log/v3"
Expand Down Expand Up @@ -300,17 +301,17 @@ func (db *DictionaryBuilder) Less(i, j int) bool {
return db.items[i].score < db.items[j].score
}

func dictionaryBuilderLess(i, j *Pattern) bool {
func dictionaryBuilderCmp(i, j *Pattern) int {
if i.score == j.score {
return bytes.Compare(i.word, j.word) < 0
return bytes.Compare(i.word, j.word)
}
return i.score < j.score
return cmp.Compare(i.score, j.score)
}

func (db *DictionaryBuilder) Swap(i, j int) {
db.items[i], db.items[j] = db.items[j], db.items[i]
}
func (db *DictionaryBuilder) Sort() { slices.SortFunc(db.items, dictionaryBuilderLess) }
func (db *DictionaryBuilder) Sort() { slices.SortFunc(db.items, dictionaryBuilderCmp) }

func (db *DictionaryBuilder) Push(x interface{}) {
db.items = append(db.items, x.(*Pattern))
Expand Down Expand Up @@ -383,11 +384,11 @@ type Pattern struct {
type PatternList []*Pattern

func (pl PatternList) Len() int { return len(pl) }
func patternListLess(i, j *Pattern) bool {
func patternListCmp(i, j *Pattern) int {
if i.uses == j.uses {
return bits.Reverse64(i.code) < bits.Reverse64(j.code)
return cmp.Compare(bits.Reverse64(i.code), bits.Reverse64(j.code))
}
return i.uses < j.uses
return cmp.Compare(i.uses, j.uses)
}

// PatternHuff is an intermediate node in a huffman tree of patterns
Expand Down Expand Up @@ -555,11 +556,11 @@ type PositionList []*Position

func (pl PositionList) Len() int { return len(pl) }

func positionListLess(i, j *Position) bool {
func positionListCmp(i, j *Position) int {
if i.uses == j.uses {
return bits.Reverse64(i.code) < bits.Reverse64(j.code)
return cmp.Compare(bits.Reverse64(i.code), bits.Reverse64(j.code))
}
return i.uses < j.uses
return cmp.Compare(i.uses, j.uses)
}

type PositionHeap []*PositionHuff
Expand All @@ -569,10 +570,14 @@ func (ph PositionHeap) Len() int {
}

func (ph PositionHeap) Less(i, j int) bool {
return ph.Compare(i, j) < 0
}

func (ph PositionHeap) Compare(i, j int) int {
if ph[i].uses == ph[j].uses {
return ph[i].tieBreaker < ph[j].tieBreaker
return cmp.Compare(ph[i].tieBreaker, ph[j].tieBreaker)
}
return ph[i].uses < ph[j].uses
return cmp.Compare(ph[i].uses, ph[j].uses)
}

func (ph *PositionHeap) Swap(i, j int) {
Expand Down
8 changes: 4 additions & 4 deletions erigon-lib/compress/parallel_compress.go
Original file line number Diff line number Diff line change
Expand Up @@ -469,7 +469,7 @@ func reducedict(ctx context.Context, trace bool, logPrefix, segmentFilePath stri
distribution[len(p.word)]++
}
}
slices.SortFunc(patternList, patternListLess)
slices.SortFunc(patternList, patternListCmp)
logCtx := make([]interface{}, 0, 8)
logCtx = append(logCtx, "patternList.Len", patternList.Len())

Expand Down Expand Up @@ -551,7 +551,7 @@ func reducedict(ctx context.Context, trace bool, logPrefix, segmentFilePath stri
}
//fmt.Printf("patternsSize = %d\n", patternsSize)
// Write all the pattens
slices.SortFunc(patternList, patternListLess)
slices.SortFunc(patternList, patternListCmp)
for _, p := range patternList {
ns := binary.PutUvarint(numBuf[:], uint64(p.depth))
if _, err = cw.Write(numBuf[:ns]); err != nil {
Expand All @@ -574,7 +574,7 @@ func reducedict(ctx context.Context, trace bool, logPrefix, segmentFilePath stri
positionList = append(positionList, p)
pos2code[pos] = p
}
slices.SortFunc(positionList, positionListLess)
slices.SortFunc(positionList, positionListCmp)
i = 0
// Build Huffman tree for codes
var posHeap PositionHeap
Expand Down Expand Up @@ -632,7 +632,7 @@ func reducedict(ctx context.Context, trace bool, logPrefix, segmentFilePath stri
}
//fmt.Printf("posSize = %d\n", posSize)
// Write all the positions
slices.SortFunc(positionList, positionListLess)
slices.SortFunc(positionList, positionListCmp)
for _, p := range positionList {
ns := binary.PutUvarint(numBuf[:], uint64(p.depth))
if _, err = cw.Write(numBuf[:ns]); err != nil {
Expand Down
16 changes: 9 additions & 7 deletions erigon-lib/downloader/snaptype/files.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@ import (
"encoding/hex"
"errors"
"fmt"
"github.com/anacrolix/torrent/metainfo"
"os"
"path/filepath"
"strconv"
"strings"

"github.com/anacrolix/torrent/metainfo"

"github.com/ledgerwatch/erigon-lib/common/cmp"
"github.com/ledgerwatch/erigon-lib/common/dir"
"golang.org/x/exp/slices"
)
Expand Down Expand Up @@ -209,20 +211,20 @@ func ParseDir(dir string) (res []FileInfo, err error) {
}
res = append(res, meta)
}
slices.SortFunc(res, func(i, j FileInfo) bool {
slices.SortFunc(res, func(i, j FileInfo) int {
if i.Version != j.Version {
return i.Version < j.Version
return cmp.Compare(i.Version, j.Version)
}
if i.From != j.From {
return i.From < j.From
return cmp.Compare(i.From, j.From)
}
if i.To != j.To {
return i.To < j.To
return cmp.Compare(i.To, j.To)
}
if i.T != j.T {
return i.T < j.T
return cmp.Compare(i.T, j.T)
}
return i.Ext < j.Ext
return cmp.Compare(i.Ext, j.Ext)
})

return res, nil
Expand Down
8 changes: 4 additions & 4 deletions erigon-lib/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ require (
github.com/stretchr/testify v1.8.4
github.com/tidwall/btree v1.6.0
golang.org/x/crypto v0.13.0
golang.org/x/exp v0.0.0-20230711023510-fffb14384f22
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
golang.org/x/sync v0.3.0
golang.org/x/sys v0.12.0
golang.org/x/time v0.3.0
Expand Down Expand Up @@ -104,10 +104,10 @@ require (
go.etcd.io/bbolt v1.3.6 // indirect
go.opentelemetry.io/otel v1.8.0 // indirect
go.opentelemetry.io/otel/trace v1.8.0 // indirect
golang.org/x/mod v0.11.0 // indirect
golang.org/x/net v0.12.0 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.14.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/tools v0.7.0 // indirect
golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
modernc.org/libc v1.22.3 // indirect
Expand Down
16 changes: 8 additions & 8 deletions erigon-lib/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -425,8 +425,8 @@ golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU
golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20230711023510-fffb14384f22 h1:FqrVOBQxQ8r/UwwXibI0KMolVhvFiGobSfdE33deHJM=
golang.org/x/exp v0.0.0-20230711023510-fffb14384f22/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ=
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
Expand All @@ -435,8 +435,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -465,8 +465,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down Expand Up @@ -545,8 +545,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E=
golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
6 changes: 3 additions & 3 deletions erigon-lib/gointerfaces/remote/sort.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ import (
"github.com/ledgerwatch/erigon-lib/gointerfaces/types"
)

func NodeInfoReplyLess(i, j *types.NodeInfoReply) bool {
func NodeInfoReplyCmp(i, j *types.NodeInfoReply) int {
if cmp := strings.Compare(i.Name, j.Name); cmp != 0 {
return cmp == -1
return cmp
}
return strings.Compare(i.Enode, j.Enode) == -1
return strings.Compare(i.Enode, j.Enode)
}
2 changes: 1 addition & 1 deletion erigon-lib/gointerfaces/remote/sort_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func TestSort(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

slices.SortFunc(tt.got.NodesInfo, remote.NodeInfoReplyLess)
slices.SortFunc(tt.got.NodesInfo, remote.NodeInfoReplyCmp)
assert.Equal(t, tt.want, tt.got)
})
}
Expand Down
3 changes: 2 additions & 1 deletion erigon-lib/patricia/patricia.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"math/bits"
"strings"

"github.com/ledgerwatch/erigon-lib/common/cmp"
"github.com/ledgerwatch/erigon-lib/sais"
"golang.org/x/exp/slices"
)
Expand Down Expand Up @@ -699,7 +700,7 @@ func (mf2 *MatchFinder2) FindLongestMatches(data []byte) []Match {
return mf2.matches
}
//sort.Sort(&mf2.matches)
slices.SortFunc(mf2.matches, func(i, j Match) bool { return i.Start < j.Start })
slices.SortFunc(mf2.matches, func(i, j Match) int { return cmp.Compare(i.Start, j.Start) })

lastEnd := mf2.matches[0].End
j := 1
Expand Down
2 changes: 1 addition & 1 deletion eth/backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -1094,7 +1094,7 @@ func (s *Ethereum) NodesInfo(limit int) (*remote.NodesInfoReply, error) {
}

nodesInfo := &remote.NodesInfoReply{NodesInfo: nodes}
slices.SortFunc(nodesInfo.NodesInfo, remote.NodeInfoReplyLess)
slices.SortFunc(nodesInfo.NodesInfo, remote.NodeInfoReplyCmp)

return nodesInfo, nil
}
Expand Down
6 changes: 3 additions & 3 deletions eth/stagedsync/stage_interhashes.go
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,7 @@ func (p *HashPromoter) Promote(logPrefix string, from, to uint64, storage bool,
}

if !storage { // delete Intermediate hashes of deleted accounts
slices.SortFunc(deletedAccounts, func(a, b []byte) bool { return bytes.Compare(a, b) < 0 })
slices.SortFunc(deletedAccounts, bytes.Compare)
for _, k := range deletedAccounts {
if err := p.tx.ForPrefix(kv.TrieOfStorage, k, func(k, v []byte) error {
if err := p.tx.Delete(kv.TrieOfStorage, k); err != nil {
Expand Down Expand Up @@ -445,7 +445,7 @@ func (p *HashPromoter) UnwindOnHistoryV3(logPrefix string, unwindFrom, unwindTo
}

// delete Intermediate hashes of deleted accounts
slices.SortFunc(deletedAccounts, func(a, b []byte) bool { return bytes.Compare(a, b) < 0 })
slices.SortFunc(deletedAccounts, bytes.Compare)
for _, k := range deletedAccounts {
if err := p.tx.ForPrefix(kv.TrieOfStorage, k, func(k, v []byte) error {
if err := p.tx.Delete(kv.TrieOfStorage, k); err != nil {
Expand Down Expand Up @@ -533,7 +533,7 @@ func (p *HashPromoter) Unwind(logPrefix string, s *StageState, u *UnwindState, s
}

if !storage { // delete Intermediate hashes of deleted accounts
slices.SortFunc(deletedAccounts, func(a, b []byte) bool { return bytes.Compare(a, b) < 0 })
slices.SortFunc(deletedAccounts, bytes.Compare)
for _, k := range deletedAccounts {
if err := p.tx.ForPrefix(kv.TrieOfStorage, k, func(k, v []byte) error {
if err := p.tx.Delete(kv.TrieOfStorage, k); err != nil {
Expand Down
Loading

0 comments on commit 408dc14

Please sign in to comment.