Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat!: Live, laugh, Shwap #3675

Open
wants to merge 102 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
102 commits
Select commit Hold shift + click to select a range
68b8b06
create store v2 file interface (#2989)
walldiss Dec 6, 2023
efde840
feat(store/mem_file): add in-memory eds file implementation (#2992)
walldiss Dec 6, 2023
806ee29
feat(share/store): Allow Store implementation to choose proof axis fo…
walldiss Mar 6, 2024
1ec1c68
chore(lint): fix gofumpt errors
walldiss May 8, 2024
b43ce4d
feat(shwap): add shwap types (#3383)
walldiss May 22, 2024
f0685c9
feat(share): Shwap types tests (#3423)
walldiss May 24, 2024
4e122f0
refactor(share/shwap): shwap types improvements
Wondertan May 27, 2024
00566a2
feat(share/store_v2) Update file interface to use shwap types (#3441)
walldiss May 29, 2024
3dd1100
refactor(shwap): Extract eds interface (#3452)
walldiss Jun 5, 2024
1c42d99
feat(share/eds) Add Accessor test infra (#3425)
walldiss Jun 10, 2024
ae28743
feat(shwap): Add close once file (#3484)
walldiss Jun 17, 2024
90beee3
add validation file
walldiss Jun 10, 2024
88178fe
update errors in sample
walldiss Jun 12, 2024
432f35d
move to eds folder
walldiss Jun 17, 2024
18ae197
cache size
walldiss Jun 17, 2024
31dfe98
use atomic cached size
walldiss Jun 17, 2024
63f6af8
fix lint
walldiss Jun 17, 2024
5585321
feat(shwap): Add ODS file (#3482)
walldiss Jun 19, 2024
b699067
test(store): add header de/serialization fuzzing (#3514)
cristaloleg Jun 21, 2024
5ef65c3
feat(share): Shwap Bitwap composition (#3421)
Wondertan Jun 26, 2024
8d850c4
feat(shwap): Add cache file (#3498)
walldiss Jun 27, 2024
807e335
feat(shwap): add Q1Q4 file (#3522)
walldiss Jun 27, 2024
660246d
feat(shwap):Add eds streaming (#3531)
walldiss Jul 2, 2024
92b2afb
feat(shwap): Separate verify and validate for shwap ids (#3551)
walldiss Jul 4, 2024
198f24e
feat(shwap): Add store cache (#3543)
walldiss Jul 4, 2024
2e80fc6
feat(shwap): Add eds store (#3545)
walldiss Jul 18, 2024
6539b26
refactor(shwap): rename share.root to axisRoots (#3583)
walldiss Jul 22, 2024
d807843
refactor(shwap): revert DataRoot -> DataHash (#3585)
Wondertan Jul 22, 2024
5aaa106
fix(shwap/bitswap): close acccessor in Blockstore (#3535)
Wondertan Jul 23, 2024
6fc490b
feat(shwap): Add namespace data shwap id (#3587)
walldiss Jul 23, 2024
c716c97
feat(shwap): Add axis roots to Accessor (#3586)
walldiss Jul 24, 2024
1bdf5f1
fix(shwap/store): ensure EDS is accessible during writes (#3596)
Wondertan Jul 25, 2024
eb8298f
feat(shwap): Add bufferisation for writing ods/Q1Q4 files (#3592)
walldiss Jul 25, 2024
c81e6d1
feat(shwap/p2p): Bitswap Getter (#3584)
Wondertan Jul 25, 2024
062f499
feat(shwap): Shwap add readFrom Writeto methods to shwap ids (#3605)
walldiss Jul 30, 2024
c4e602a
feat(shwap): Integrate shrex into shwap (#3554)
walldiss Jul 30, 2024
8912fc0
feat(shwap): Shwap core integration (#3598)
walldiss Jul 31, 2024
42913ea
feat(shwap): Shwap cascade and mock getters (#3599)
walldiss Jul 31, 2024
8d5ae0a
refactor(shwap): Rename remaining dataroot references (#3607)
walldiss Jul 31, 2024
9118b8e
fix(shwap): don't cache empty eds and close recent accessor (#3608)
walldiss Jul 31, 2024
dfaffb7
chore(shwap): various improvements (#3609)
Wondertan Aug 1, 2024
df13c2a
feat(shwap): Update availability to work with shwap (#3600)
walldiss Aug 1, 2024
d6d292a
feat(shwap): add shwap to blob module (#3613)
walldiss Aug 1, 2024
6751df2
feat(shwap): update daser for shwap (#3614)
walldiss Aug 1, 2024
5bb844a
feat(shwap): add shwap store getter (#3617)
walldiss Aug 7, 2024
5f6fdab
feat(shwap): Add caching to blockstore (#3615)
walldiss Aug 7, 2024
7e6f0a5
feat(shwap/bitswap): Shwap optimized Bitswap constructors (#3536)
Wondertan Aug 8, 2024
36bb8e6
feat(shwap): Improve eds store cache refcounter forgot Close() protec…
walldiss Aug 8, 2024
fca7ff7
fix(shwap/cahce): use local copy of axis in proofs cache (#3657)
walldiss Aug 13, 2024
3026164
refactor(shwap/store): simplify and cleanup read logic
Wondertan Aug 9, 2024
ef56ae8
chore(shwap|share): add empty EDS helpers
Wondertan Aug 14, 2024
4b6f7c2
refactor!(shwap/store): decouple ODS and Q1Q4 files
Wondertan Aug 11, 2024
ed1e80a
perf(shwap/store): ODSQ4 lazy loading
Wondertan Aug 13, 2024
5395c13
refactor(shwap/store): make empty block a file
Wondertan Aug 13, 2024
0f2af92
chore(shwap/store): fix logger name
Wondertan Aug 13, 2024
12d5c2d
chore(shwap/store): update file benchmarks
Wondertan Aug 14, 2024
9449437
perf(shwap/store): create ODS and Q4 files in parallel
Wondertan Aug 14, 2024
965f1cc
feat(shwap): shwap nodebuilder and cleanup (#3637)
walldiss Aug 15, 2024
5d6ab85
refactor(shwap): extract ReadAccessor
Wondertan Aug 3, 2024
34da644
test(shwap/eds): tail padded EDS in accessor suite
Wondertan Aug 15, 2024
01b1655
feat(shwap): ODS tail padding trim
Wondertan Aug 15, 2024
0c88b31
Merge branch 'refs/heads/main-celestia' into shwap-rebase
walldiss Aug 20, 2024
a3531ce
resolve conflicts
walldiss Aug 20, 2024
0bfff96
chore(shwap)!: Merge main into shwap (#3662)
walldiss Aug 22, 2024
dd7b5cc
fix(store/files): ODSQ4 Closing audit (#3691)
Wondertan Aug 26, 2024
568744f
perf(store/sync): avoid syncing files (#3693)
Wondertan Aug 26, 2024
e1d888c
refactor(store/file): auto-detect q4 and remove fileType from the fil…
walldiss Aug 26, 2024
623d8cc
refactor(store/file): increase header size to 64
walldiss Aug 26, 2024
393f131
refactor(store/file): remove header from Q4
walldiss Aug 26, 2024
8d3e50c
refactor(store): remove link to Q4 file and find it by datahash
walldiss Aug 26, 2024
c63353c
fix(store/file): use proper constructor for odsQ4 tests
walldiss Aug 27, 2024
e93e63f
refactor(store/file): remove Offset method from header
walldiss Aug 27, 2024
ed82a2e
fix(shwap): io.ReadFull for read helpers (#3696)
Wondertan Aug 28, 2024
1502153
fix(shwap): io.ReadFull for NamespaceDataID (#3697)
walldiss Aug 28, 2024
a58b3bf
feat(shwap): Add discovery version (#3616)
walldiss Sep 11, 2024
d1d37ee
fix(shwap/bitswap): correct protocol id (#3722)
Wondertan Sep 11, 2024
5928c6a
feat(cmd/cel-shed): shwap (#3720)
Wondertan Sep 11, 2024
efa7365
feat(shwap/store): Add q4 trimming support in store (#3680)
walldiss Sep 11, 2024
51c3f94
Merge branch 'main' into shwap
Wondertan Sep 12, 2024
75de820
chore: bump libp2p-pubsub
Wondertan Sep 12, 2024
983e9c0
feat(shwap): Equals methods for IDs
Wondertan Sep 13, 2024
1a70470
chore(shwap/bitswap): check ID equality
Wondertan Sep 13, 2024
f70bccc
chore(shwap/bitswap): address naming nit
Wondertan Sep 13, 2024
15ebe17
Revert "chore: bump libp2p-pubsub"
Wondertan Sep 16, 2024
e6d3a58
fix(store/odsq4): fix order of opened flag stored (#3735)
walldiss Sep 17, 2024
e1c7403
fix(store/ods): fix full ods caching (#3736)
walldiss Sep 17, 2024
078f9ab
feat(nodebuilder): Blockstore cache params (#3739)
walldiss Sep 17, 2024
8f6c920
remove new_eds folder
walldiss Sep 17, 2024
1cf355d
refactor(share/eds): various accessor testing suite improvements (#3737)
walldiss Sep 17, 2024
1279c2e
fix(core): fix flaky generateNonEmptyBlocks (#3740)
walldiss Sep 17, 2024
89e8781
fix(blob): fix blobsub subscription cancellation test (#3746)
walldiss Sep 18, 2024
2c67306
fix(store): use symlinks for empty blocks heighs (#3747)
walldiss Sep 18, 2024
7ffd7c0
chore(shwap/bitswap): prevent not found error logging (#3754)
Wondertan Sep 20, 2024
8ba3dad
chore(shwap/bitswap): add getter tracing (#3755)
Wondertan Sep 20, 2024
4289846
feat(share/availability): Dont store q4 on archival (#3751)
walldiss Sep 20, 2024
53ef4d9
Reapply "chore: bump libp2p-pubsub"
Wondertan Sep 20, 2024
0584f6a
perf(core): increase concurrency limit
Wondertan Sep 18, 2024
71af216
feat(store/cache): add Has methods (#3757)
Wondertan Sep 20, 2024
3031e86
chore(shwap/bitswap): remove spammy log
Wondertan Sep 21, 2024
572c621
chore(shwap): Validate -> Verify
Wondertan Sep 21, 2024
f48353c
Merge branch 'main' into shwap
Wondertan Sep 23, 2024
c5c959f
fix(nodebuidlder/p2p) Use semver for discovery prefix (#3768)
walldiss Sep 24, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat(store/mem_file): add in-memory eds file implementation (#2992)
Adds in-memory file implementation
  • Loading branch information
walldiss committed Jul 6, 2024
commit efde8404355c8d6f523617d22e791d8a2e208c27
6 changes: 4 additions & 2 deletions share/eds/edstest/testing.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,14 @@ func RandEDS(t testing.TB, size int) *rsmt2d.ExtendedDataSquare {
return eds
}

// RandEDSWithNamespace generates EDS with given square size. Returned EDS will have namespacedAmount of
// shares with the given namespace.
func RandEDSWithNamespace(
t testing.TB,
namespace share.Namespace,
size int,
namespacedAmount, size int,
) (*rsmt2d.ExtendedDataSquare, *share.Root) {
shares := sharetest.RandSharesWithNamespace(t, namespace, size*size)
shares := sharetest.RandSharesWithNamespace(t, namespace, namespacedAmount, size*size)
eds, err := rsmt2d.ComputeExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(size)))
require.NoError(t, err, "failure to recompute the extended data square")
dah, err := share.NewRoot(eds)
Expand Down
9 changes: 6 additions & 3 deletions share/getter.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,16 +71,19 @@ func (ns NamespacedShares) Verify(root *Root, namespace Namespace) error {
}

for i, row := range ns {
if row.Proof == nil && row.Shares == nil {
return fmt.Errorf("row verification failed: no proofs and shares")
}
// verify row data against row hash from original root
if !row.verify(originalRoots[i], namespace) {
if !row.Verify(originalRoots[i], namespace) {
return fmt.Errorf("row verification failed: row %d doesn't match original root: %s", i, root.String())
}
}
return nil
}

// verify validates the row using nmt inclusion proof.
func (row *NamespacedRow) verify(rowRoot []byte, namespace Namespace) bool {
// Verify validates the row using nmt inclusion proof.
func (row *NamespacedRow) Verify(rowRoot []byte, namespace Namespace) bool {
// construct nmt leaves from shares by prepending namespace
leaves := make([][]byte, 0, len(row.Shares))
for _, shr := range row.Shares {
Expand Down
3 changes: 2 additions & 1 deletion share/getters/shrex_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,9 @@ func TestShrexGetter(t *testing.T) {
t.Cleanup(cancel)

// generate test data
size := 64
namespace := sharetest.RandV0Namespace()
randEDS, dah := edstest.RandEDSWithNamespace(t, namespace, 64)
randEDS, dah := edstest.RandEDSWithNamespace(t, namespace, size*size, size)
eh := headertest.RandExtendedHeaderWithRoot(t, dah)
require.NoError(t, edsStore.Put(ctx, dah.Hash(), randEDS))
fullPeerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{
Expand Down
13 changes: 11 additions & 2 deletions share/sharetest/testing.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,17 +38,26 @@ func RandShares(t testing.TB, total int) []share.Share {
}

// RandSharesWithNamespace is same the as RandShares, but sets same namespace for all shares.
func RandSharesWithNamespace(t testing.TB, namespace share.Namespace, total int) []share.Share {
func RandSharesWithNamespace(t testing.TB, namespace share.Namespace, namespacedAmount, total int) []share.Share {
if total&(total-1) != 0 {
t.Errorf("total must be power of 2: %d", total)
t.FailNow()
}

if namespacedAmount > total {
t.Errorf("withNamespace must be less than total: %d", total)
t.FailNow()
}

shares := make([]share.Share, total)
rnd := rand.New(rand.NewSource(time.Now().Unix())) //nolint:gosec
for i := range shares {
shr := make([]byte, share.Size)
copy(share.GetNamespace(shr), namespace)
if i < namespacedAmount {
copy(share.GetNamespace(shr), namespace)
} else {
copy(share.GetNamespace(shr), RandV0Namespace())
}
_, err := rnd.Read(share.GetData(shr))
require.NoError(t, err)
shares[i] = shr
Expand Down
2 changes: 1 addition & 1 deletion share/store/eds_file.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ type EdsFile interface {
// AxisHalf returns shares for the first half of the axis of the given type and index.
AxisHalf(ctx context.Context, axisType rsmt2d.Axis, axisIdx int) ([]share.Share, error)
// Data returns data for the given namespace and row index.
Data(ctx context.Context, namespace share.Namespace, rowIdx int) ([]share.NamespacedRow, error)
Data(ctx context.Context, namespace share.Namespace, rowIdx int) (share.NamespacedRow, error)
// EDS returns extended data square stored in the file.
EDS(ctx context.Context) (*rsmt2d.ExtendedDataSquare, error)
}
101 changes: 101 additions & 0 deletions share/store/mem_file.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package store

import (
"context"

"github.com/celestiaorg/celestia-app/pkg/wrapper"
"github.com/celestiaorg/nmt"
"github.com/celestiaorg/rsmt2d"

"github.com/celestiaorg/celestia-node/share"
"github.com/celestiaorg/celestia-node/share/ipld"
)

var _ EdsFile = (*MemFile)(nil)

type MemFile struct {
Eds *rsmt2d.ExtendedDataSquare
}

func (f *MemFile) Close() error {
return nil
}

func (f *MemFile) Size() int {
return int(f.Eds.Width())
}

func (f *MemFile) Share(
_ context.Context,
axisType rsmt2d.Axis,
axisIdx, shrIdx int,
) (share.Share, nmt.Proof, error) {
shares := f.axis(axisType, axisIdx)
tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(f.Size()/2), uint(axisIdx))
for _, shr := range shares {
err := tree.Push(shr)
if err != nil {
return nil, nmt.Proof{}, err
}
}

proof, err := tree.ProveRange(shrIdx, shrIdx+1)
if err != nil {
return nil, nmt.Proof{}, err
}

return shares[shrIdx], proof, nil
}

func (f *MemFile) AxisHalf(_ context.Context, axisType rsmt2d.Axis, axisIdx int) ([]share.Share, error) {
return f.axis(axisType, axisIdx)[:f.Size()/2], nil
}

func (f *MemFile) Data(_ context.Context, namespace share.Namespace, rowIdx int) (share.NamespacedRow, error) {
shares := f.axis(rsmt2d.Row, rowIdx)
bserv := ipld.NewMemBlockservice()
batchAdder := ipld.NewNmtNodeAdder(context.TODO(), bserv, ipld.MaxSizeBatchOption(len(shares)))
tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(len(shares)/2), uint(rowIdx),
nmt.NodeVisitor(batchAdder.Visit))
for _, shr := range shares {
err := tree.Push(shr)
if err != nil {
return share.NamespacedRow{}, err
}
}

root, err := tree.Root()
if err != nil {
return share.NamespacedRow{}, err
}

err = batchAdder.Commit()
if err != nil {
return share.NamespacedRow{}, err
}

cid := ipld.MustCidFromNamespacedSha256(root)
row, proof, err := ipld.GetSharesByNamespace(context.TODO(), bserv, cid, namespace, len(shares))
if err != nil {
return share.NamespacedRow{}, err
}
return share.NamespacedRow{
Shares: row,
Proof: proof,
}, nil
}

func (f *MemFile) EDS(_ context.Context) (*rsmt2d.ExtendedDataSquare, error) {
return f.Eds, nil
}

func (f *MemFile) axis(axisType rsmt2d.Axis, axisIdx int) []share.Share {
switch axisType {
case rsmt2d.Row:
return f.Eds.Row(uint(axisIdx))
case rsmt2d.Col:
return f.Eds.Col(uint(axisIdx))
default:
panic("unknown axis")
}
}
69 changes: 69 additions & 0 deletions share/store/mem_file_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package store

import (
"context"
"crypto/sha256"
mrand "math/rand"
"testing"

"github.com/stretchr/testify/require"

"github.com/celestiaorg/rsmt2d"

"github.com/celestiaorg/celestia-node/share"
"github.com/celestiaorg/celestia-node/share/eds/edstest"
"github.com/celestiaorg/celestia-node/share/sharetest"
)

func TestMemFileShare(t *testing.T) {
eds := edstest.RandEDS(t, 32)
root, err := share.NewRoot(eds)
require.NoError(t, err)
fl := &MemFile{Eds: eds}

width := int(eds.Width())
for _, axisType := range []rsmt2d.Axis{rsmt2d.Col, rsmt2d.Row} {
for i := 0; i < width*width; i++ {
axisIdx, shrIdx := i/width, i%width
if axisType == rsmt2d.Col {
axisIdx, shrIdx = shrIdx, axisIdx
}

shr, prf, err := fl.Share(context.TODO(), axisType, axisIdx, shrIdx)
require.NoError(t, err)

namespace := share.ParitySharesNamespace
if axisIdx < width/2 && shrIdx < width/2 {
namespace = share.GetNamespace(shr)
}

axishash := root.RowRoots[axisIdx]
if axisType == rsmt2d.Col {
axishash = root.ColumnRoots[axisIdx]
}

ok := prf.VerifyInclusion(sha256.New(), namespace.ToNMT(), [][]byte{shr}, axishash)
require.True(t, ok)
}
}
}

func TestMemFileDate(t *testing.T) {
size := 32

// generate EDS with random data and some shares with the same namespace
namespace := sharetest.RandV0Namespace()
amount := mrand.Intn(size*size-1) + 1
eds, dah := edstest.RandEDSWithNamespace(t, namespace, amount, size)

file := &MemFile{Eds: eds}

for i, root := range dah.RowRoots {
if !namespace.IsOutsideRange(root, root) {
nd, err := file.Data(context.Background(), namespace, i)
require.NoError(t, err)
ok := nd.Verify(root, namespace)
require.True(t, ok)
}
}
}