Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Shwap #2675

Draft
wants to merge 32 commits into
base: main
Choose a base branch
from
Draft

Shwap #2675

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
d953117
chore(moddas): speed up sampling for LN
Wondertan Nov 29, 2023
f122a71
prototype
Wondertan Sep 7, 2023
7237ad7
feat(modp2p): listen on WebTransport by default
Wondertan Sep 11, 2023
ab6bec8
lint
Wondertan Sep 12, 2023
2f08bdd
now test verifies all the share proofs
Wondertan Sep 17, 2023
b33515e
refactor sampling protocol and use proto for serialization
Wondertan Sep 18, 2023
59aa730
docs and tests for ipldv2
Wondertan Sep 18, 2023
1b3d881
add support for col proofs sampling
Wondertan Sep 21, 2023
b53769b
blockstore impl and various cleanups and improvements
Wondertan Sep 22, 2023
ce31854
initial support for ODS Mode
Wondertan Sep 22, 2023
3ab6b37
implement axis sampling
Wondertan Sep 30, 2023
041ed3d
introduce File interface and decouple ipldv2 tests from on disk file
Wondertan Oct 1, 2023
1601460
use height as block id
Wondertan Oct 19, 2023
600d186
chore: extract proto helper
Wondertan Oct 19, 2023
6673564
successful experiment with request size shortening for axis sampling
Wondertan Oct 19, 2023
830860d
docs fix
Wondertan Oct 19, 2023
9ffb284
request size optimization for share sample
Wondertan Oct 19, 2023
52f3ab9
refactor AxisID away and many more improvements
Wondertan Oct 22, 2023
21bd2fc
remove serialization ambigiouty and ensure there is only one serializ…
Wondertan Oct 22, 2023
cdbd694
cleanup proto field names
Wondertan Oct 22, 2023
c3e8450
namespace mh
Wondertan Dec 2, 2023
8a66fd5
namespace mh but finished and tested
Wondertan Dec 2, 2023
bbcd956
lol
Wondertan Dec 2, 2023
583481b
pass by value and cid must constructors
Wondertan Dec 3, 2023
a28cfef
fix data id test
Wondertan Dec 3, 2023
f6db8f9
blockservice constructor
Wondertan Dec 3, 2023
9a8b5ed
implement Getter and tests for it
Wondertan Dec 3, 2023
0d4dd27
rename to shwap
Wondertan Dec 6, 2023
c96579a
ensure only shares a cached in blockstore
Wondertan Dec 6, 2023
bb034b3
add sessions
Wondertan Dec 6, 2023
fc082f4
protocol updates:
Wondertan Dec 28, 2023
a358713
remove ids
Wondertan Mar 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add support for col proofs sampling
  • Loading branch information
@Wondertan
Wondertan committed Dec 2, 2023
commit 1b3d881693f65d990fdd029687eb731863c928f6
58 changes: 40 additions & 18 deletions share/eds/file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,23 +86,42 @@ func (f *File) Header() Header {
return f.hdr
}

func (f *File) Axis(idx int, _ rsmt2d.Axis) ([]share.Share, error) {
// TODO: Add Col support
shrLn := int64(f.hdr.ShareSize)
sqrLn := int64(f.hdr.SquareSize)
rwwLn := shrLn * sqrLn
func (f *File) Axis(idx int, axis rsmt2d.Axis) ([]share.Share, error) {
shrLn := int(f.hdr.ShareSize)
sqrLn := int(f.hdr.SquareSize)

offset := int64(idx)*rwwLn + HeaderSize
rowdata := make([]byte, rwwLn)
if _, err := f.fl.ReadAt(rowdata, offset); err != nil {
return nil, err
}
shrs := make([]share.Share, sqrLn)
switch axis {
case rsmt2d.Col:
// [] [] [] []
// [] [] [] []
// [] [] [] []
// [] [] [] []

for i := 0; i < sqrLn; i++ {
pos := idx + i*sqrLn
offset := pos*shrLn + HeaderSize

shr := make(share.Share, shrLn)
if _, err := f.fl.ReadAt(shr, int64(offset)); err != nil {
return nil, err
}
shrs[i] = shr
}
case rsmt2d.Row:
pos := idx * sqrLn
offset := pos*shrLn + HeaderSize
axsData := make([]byte, sqrLn*shrLn)
if _, err := f.fl.ReadAt(axsData, int64(offset)); err != nil {
return nil, err
}

row := make([]share.Share, sqrLn)
for i := range row {
row[i] = rowdata[int64(i)*shrLn : (int64(i)+1)*shrLn]
for i := range shrs {
shrs[i] = axsData[i*shrLn : (i+1)*shrLn]
}
}
return row, nil

return shrs, nil
}

func (f *File) Share(idx int) (share.Share, error) {
Expand All @@ -120,21 +139,24 @@ func (f *File) Share(idx int) (share.Share, error) {
func (f *File) ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof, error) {
Copy link
Contributor

@derrandz derrandz Sep 19, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[feature request]: Having AxisWithProof will be super useful for search operations to retrieve all rows with their proofs and roots to be able to filter out for specific shares and their proofs by roots

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When you get the whole axis, there is no proof.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I mean, if you look at ShareWithProof, to generate a proof for that share, you have to get the whole axis, right? Retrieving an entire axis and recomputing its proof would be sufficient for the purposes I mentioned.

// TODO: Cache the axis as well as computed tree
sqrLn := int(f.hdr.SquareSize)
rowIdx := idx / sqrLn
shrs, err := f.Axis(rowIdx, axis)
axsIdx, shrIdx := idx/sqrLn, idx%sqrLn
if axis == rsmt2d.Col {
axsIdx, shrIdx = shrIdx, axsIdx
}

shrs, err := f.Axis(axsIdx, axis)
if err != nil {
return nil, nmt.Proof{}, err
}

tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(sqrLn/2), uint(rowIdx))
tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(sqrLn/2), uint(axsIdx))
for _, shr := range shrs {
err = tree.Push(shr)
if err != nil {
return nil, nmt.Proof{}, err
}
}

shrIdx := idx % sqrLn
proof, err := tree.ProveRange(shrIdx, shrIdx+1)
if err != nil {
return nil, nmt.Proof{}, err
Expand Down
69 changes: 45 additions & 24 deletions share/eds/file_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@ import (

func TestFile(t *testing.T) {
path := t.TempDir() + "/testfile"
eds := edstest.RandEDS(t, 16)
eds := edstest.RandEDS(t, 8)
root, err := share.NewRoot(eds)
require.NoError(t, err)

fl, err := CreateFile(path, eds)
require.NoError(t, err)
Expand All @@ -25,33 +27,40 @@ func TestFile(t *testing.T) {
fl, err = OpenFile(path)
require.NoError(t, err)

for i := 0; i < int(eds.Width()); i++ {
row, err := fl.Axis(i, rsmt2d.Row)
require.NoError(t, err)
assert.EqualValues(t, eds.Row(uint(i)), row)
axis := []rsmt2d.Axis{rsmt2d.Col, rsmt2d.Row}
for _, axis := range axis {
for i := 0; i < int(eds.Width()); i++ {
row, err := fl.Axis(i, axis)
require.NoError(t, err)
assert.EqualValues(t, getAxis(i, axis, eds), row)
}
}

width := int(eds.Width())
for i := 0; i < width*width; i++ {
row, col := uint(i/width), uint(i%width)
shr, err := fl.Share(i)
require.NoError(t, err)
assert.EqualValues(t, eds.GetCell(row, col), shr)

shr, proof, err := fl.ShareWithProof(i, rsmt2d.Row)
require.NoError(t, err)
assert.EqualValues(t, eds.GetCell(row, col), shr)

roots, err := eds.RowRoots()
require.NoError(t, err)

namespace := share.ParitySharesNamespace
if int(row) < width/2 && int(col) < width/2 {
namespace = share.GetNamespace(shr)
}
for _, axis := range axis {
for i := 0; i < width*width; i++ {
row, col := uint(i/width), uint(i%width)
shr, err := fl.Share(i)
require.NoError(t, err)
assert.EqualValues(t, eds.GetCell(row, col), shr)

shr, proof, err := fl.ShareWithProof(i, axis)
require.NoError(t, err)
assert.EqualValues(t, eds.GetCell(row, col), shr)

namespace := share.ParitySharesNamespace
if int(row) < width/2 && int(col) < width/2 {
namespace = share.GetNamespace(shr)
}

ok := proof.VerifyInclusion(sha256.New(), namespace.ToNMT(), [][]byte{shr}, roots[row])
assert.True(t, ok)
dahroot := root.RowRoots[row]
if axis == rsmt2d.Col {
dahroot = root.ColumnRoots[col]
}

ok := proof.VerifyInclusion(sha256.New(), namespace.ToNMT(), [][]byte{shr}, dahroot)
assert.True(t, ok)
}
}

out, err := fl.EDS()
Expand All @@ -61,3 +70,15 @@ func TestFile(t *testing.T) {
err = fl.Close()
require.NoError(t, err)
}

// TODO(@Wondertan): Should be a method on eds
func getAxis(idx int, axis rsmt2d.Axis, eds *rsmt2d.ExtendedDataSquare) [][]byte {
switch axis {
case rsmt2d.Row:
return eds.Row(uint(idx))
case rsmt2d.Col:
return eds.Col(uint(idx))
default:
panic("")
}
}
29 changes: 16 additions & 13 deletions share/ipldv2/ipldv2_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,24 +26,27 @@ func TestV2Roundtrip(t *testing.T) {
dn.ConnectAll()

square := edstest.RandEDS(t, 16)
axis := []rsmt2d.Axis{rsmt2d.Col, rsmt2d.Row}
width := int(square.Width())
for i := 0; i < width*width; i++ {
smpl, err := NewSampleFrom(square, i, rsmt2d.Row)
require.NoError(t, err)
for _, axis := range axis {
for i := 0; i < width*width; i++ {
smpl, err := NewSampleFrom(square, i, axis)
require.NoError(t, err)

err = smpl.Validate()
require.NoError(t, err)
err = smpl.Validate()
require.NoError(t, err)

blkIn, err := smpl.IPLDBlock()
require.NoError(t, err)
blkIn, err := smpl.IPLDBlock()
require.NoError(t, err)

err = srv1.AddBlock(ctx, blkIn)
require.NoError(t, err)
err = srv1.AddBlock(ctx, blkIn)
require.NoError(t, err)

blkOut, err := srv2.GetBlock(ctx, blkIn.Cid())
require.NoError(t, err)
blkOut, err := srv2.GetBlock(ctx, blkIn.Cid())
require.NoError(t, err)

assert.EqualValues(t, blkIn.RawData(), blkOut.RawData())
assert.EqualValues(t, blkIn.Cid(), blkOut.Cid())
assert.EqualValues(t, blkIn.RawData(), blkOut.RawData())
assert.EqualValues(t, blkIn.Cid(), blkOut.Cid())
}
}
}
17 changes: 14 additions & 3 deletions share/ipldv2/sample.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,15 +59,26 @@ func NewSample(root *share.Root, idx int, axis rsmt2d.Axis, shr share.Share, pro
// NewSampleFrom samples the EDS and constructs a new Sample.
func NewSampleFrom(eds *rsmt2d.ExtendedDataSquare, idx int, axis rsmt2d.Axis) (*Sample, error) {
sqrLn := int(eds.Width())
rowIdx, shrIdx := idx/sqrLn, idx%sqrLn
shrs := eds.Row(uint(rowIdx))
axisIdx, shrIdx := idx/sqrLn, idx%sqrLn

// TODO(@Wondertan): Should be an rsmt2d method
var shrs [][]byte
switch axis {
case rsmt2d.Row:
shrs = eds.Row(uint(axisIdx))
case rsmt2d.Col:
axisIdx, shrIdx = shrIdx, axisIdx
shrs = eds.Col(uint(axisIdx))
default:
panic("invalid axis")
}

root, err := share.NewRoot(eds)
if err != nil {
return nil, err
}

tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(sqrLn/2), uint(rowIdx))
tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(sqrLn/2), uint(axisIdx))
for _, shr := range shrs {
err := tree.Push(shr)
if err != nil {
Expand Down