From a2df5d6b22a54d391e7d902a3ab00c8062e51183 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Wed, 22 May 2024 14:24:33 +0300 Subject: [PATCH 1/2] update file interface to use shwap types --- share/eds/utils.go | 14 ++--- share/ipld/get_shares.go | 5 +- share/ipld/get_shares_test.go | 10 ++-- share/root.go | 51 ++++++++++++++++ share/share.go | 48 --------------- share/store/file/axis_half.go | 19 ++++++ share/store/{ => file}/eds_file.go | 11 ++-- share/store/{ => file}/mem_file.go | 77 ++++++++++++++----------- share/store/{ => file}/mem_file_test.go | 17 +++--- 9 files changed, 140 insertions(+), 112 deletions(-) create mode 100644 share/store/file/axis_half.go rename share/store/{ => file}/eds_file.go (71%) rename share/store/{ => file}/mem_file.go (60%) rename share/store/{ => file}/mem_file_test.go (76%) diff --git a/share/eds/utils.go b/share/eds/utils.go index b897dd14b5..c38b74e349 100644 --- a/share/eds/utils.go +++ b/share/eds/utils.go @@ -121,24 +121,24 @@ func CollectSharesByNamespace( utils.SetStatusAndEnd(span, err) }() - rootCIDs := ipld.FilterRootByNamespace(root, namespace) - if len(rootCIDs) == 0 { + rowIdxs := share.RowsWithNamespace(root, namespace) + if len(rowIdxs) == 0 { return []share.NamespacedRow{}, nil } errGroup, ctx := errgroup.WithContext(ctx) - shares = make([]share.NamespacedRow, len(rootCIDs)) - for i, rootCID := range rootCIDs { + shares = make([]share.NamespacedRow, len(rowIdxs)) + for i, rowIdx := range rowIdxs { // shadow loop variables, to ensure correct values are captured - i, rootCID := i, rootCID + rowIdx, rowRoot := rowIdx, root.RowRoots[rowIdx] errGroup.Go(func() error { - row, proof, err := ipld.GetSharesByNamespace(ctx, bg, rootCID, namespace, len(root.RowRoots)) + row, proof, err := ipld.GetSharesByNamespace(ctx, bg, rowRoot, namespace, len(root.RowRoots)) shares[i] = share.NamespacedRow{ Shares: row, Proof: proof, } if err != nil { - return fmt.Errorf("retrieving shares by namespace %s for row %x: %w", namespace.String(), rootCID, err) + return fmt.Errorf("retrieving shares by namespace %s for row %d: %w", namespace.String(), rowIdx, err) } return nil }) diff --git a/share/ipld/get_shares.go b/share/ipld/get_shares.go index 98db7012b5..2883e62761 100644 --- a/share/ipld/get_shares.go +++ b/share/ipld/get_shares.go @@ -44,12 +44,13 @@ func GetShares(ctx context.Context, bg blockservice.BlockGetter, root cid.Cid, s func GetSharesByNamespace( ctx context.Context, bGetter blockservice.BlockGetter, - root cid.Cid, + root []byte, namespace share.Namespace, maxShares int, ) ([]share.Share, *nmt.Proof, error) { + rootCid := MustCidFromNamespacedSha256(root) data := NewNamespaceData(maxShares, namespace, WithLeaves(), WithProofs()) - err := data.CollectLeavesByNamespace(ctx, bGetter, root) + err := data.CollectLeavesByNamespace(ctx, bGetter, rootCid) if err != nil { return nil, nil, err } diff --git a/share/ipld/get_shares_test.go b/share/ipld/get_shares_test.go index 2f5d630473..c5b606acee 100644 --- a/share/ipld/get_shares_test.go +++ b/share/ipld/get_shares_test.go @@ -174,8 +174,7 @@ func TestGetSharesByNamespace(t *testing.T) { rowRoots, err := eds.RowRoots() require.NoError(t, err) for _, row := range rowRoots { - rcid := MustCidFromNamespacedSha256(row) - rowShares, _, err := GetSharesByNamespace(ctx, bServ, rcid, namespace, len(rowRoots)) + rowShares, _, err := GetSharesByNamespace(ctx, bServ, row, namespace, len(rowRoots)) if errors.Is(err, ErrNamespaceOutsideRange) { continue } @@ -363,8 +362,7 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { rowRoots, err := eds.RowRoots() require.NoError(t, err) for _, row := range rowRoots { - rcid := MustCidFromNamespacedSha256(row) - rowShares, proof, err := GetSharesByNamespace(ctx, bServ, rcid, namespace, len(rowRoots)) + rowShares, proof, err := GetSharesByNamespace(ctx, bServ, row, namespace, len(rowRoots)) if namespace.IsOutsideRange(row, row) { require.ErrorIs(t, err, ErrNamespaceOutsideRange) continue @@ -386,7 +384,7 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { share.NewSHA256Hasher(), namespace.ToNMT(), leaves, - NamespacedSha256FromCID(rcid)) + row) require.True(t, verified) // verify inclusion @@ -394,7 +392,7 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { share.NewSHA256Hasher(), namespace.ToNMT(), rowShares, - NamespacedSha256FromCID(rcid)) + row) require.True(t, verified) } } diff --git a/share/root.go b/share/root.go index 59d7a0a177..8b14b9979e 100644 --- a/share/root.go +++ b/share/root.go @@ -1,6 +1,12 @@ package share import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "hash" + "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/rsmt2d" ) @@ -9,6 +15,30 @@ import ( // In practice, it is a commitment to all the Data in a square. type Root = da.DataAvailabilityHeader +// DataHash is a representation of the Root hash. +type DataHash []byte + +func (dh DataHash) Validate() error { + if len(dh) != 32 { + return fmt.Errorf("invalid hash size, expected 32, got %d", len(dh)) + } + return nil +} + +func (dh DataHash) String() string { + return fmt.Sprintf("%X", []byte(dh)) +} + +// IsEmptyRoot check whether DataHash corresponds to the root of an empty block EDS. +func (dh DataHash) IsEmptyRoot() bool { + return bytes.Equal(EmptyRoot().Hash(), dh) +} + +// NewSHA256Hasher returns a new instance of a SHA-256 hasher. +func NewSHA256Hasher() hash.Hash { + return sha256.New() +} + // NewRoot generates Root(DataAvailabilityHeader) using the // provided extended data square. func NewRoot(eds *rsmt2d.ExtendedDataSquare) (*Root, error) { @@ -29,3 +59,24 @@ func RowsWithNamespace(root *Root, namespace Namespace) (idxs []int) { } return } + +// RootHashForCoordinates returns the root hash for the given coordinates. +func RootHashForCoordinates(r *Root, axisType rsmt2d.Axis, rowIdx, colIdx uint) []byte { + if axisType == rsmt2d.Row { + return r.RowRoots[rowIdx] + } + return r.ColumnRoots[colIdx] +} + +// MustDataHashFromString converts a hex string to a valid datahash. +func MustDataHashFromString(datahash string) DataHash { + dh, err := hex.DecodeString(datahash) + if err != nil { + panic(fmt.Sprintf("datahash conversion: passed string was not valid hex: %s", datahash)) + } + err = DataHash(dh).Validate() + if err != nil { + panic(fmt.Sprintf("datahash validation: passed hex string failed: %s", err)) + } + return dh +} diff --git a/share/share.go b/share/share.go index e2f351e805..b97bab3589 100644 --- a/share/share.go +++ b/share/share.go @@ -1,11 +1,8 @@ package share import ( - "bytes" "crypto/sha256" - "encoding/hex" "fmt" - "hash" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/nmt" @@ -72,48 +69,3 @@ func (s *ShareWithProof) Validate(rootHash []byte, x, y, edsSize int) bool { rootHash, ) } - -// DataHash is a representation of the Root hash. -type DataHash []byte - -func (dh DataHash) Validate() error { - if len(dh) != 32 { - return fmt.Errorf("invalid hash size, expected 32, got %d", len(dh)) - } - return nil -} - -func (dh DataHash) String() string { - return fmt.Sprintf("%X", []byte(dh)) -} - -// IsEmptyRoot check whether DataHash corresponds to the root of an empty block EDS. -func (dh DataHash) IsEmptyRoot() bool { - return bytes.Equal(EmptyRoot().Hash(), dh) -} - -// MustDataHashFromString converts a hex string to a valid datahash. -func MustDataHashFromString(datahash string) DataHash { - dh, err := hex.DecodeString(datahash) - if err != nil { - panic(fmt.Sprintf("datahash conversion: passed string was not valid hex: %s", datahash)) - } - err = DataHash(dh).Validate() - if err != nil { - panic(fmt.Sprintf("datahash validation: passed hex string failed: %s", err)) - } - return dh -} - -// NewSHA256Hasher returns a new instance of a SHA-256 hasher. -func NewSHA256Hasher() hash.Hash { - return sha256.New() -} - -// RootHashForCoordinates returns the root hash for the given coordinates. -func RootHashForCoordinates(r *Root, axisType rsmt2d.Axis, rowIdx, colIdx uint) []byte { - if axisType == rsmt2d.Row { - return r.RowRoots[rowIdx] - } - return r.ColumnRoots[colIdx] -} diff --git a/share/store/file/axis_half.go b/share/store/file/axis_half.go new file mode 100644 index 0000000000..58b9306b85 --- /dev/null +++ b/share/store/file/axis_half.go @@ -0,0 +1,19 @@ +package file + +import ( + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/shwap" +) + +type AxisHalf struct { + Shares []share.Share + IsParity bool +} + +func (a AxisHalf) ToRow() shwap.Row { + side := shwap.Left + if a.IsParity { + side = shwap.Right + } + return shwap.NewRow(a.Shares, side) +} diff --git a/share/store/eds_file.go b/share/store/file/eds_file.go similarity index 71% rename from share/store/eds_file.go rename to share/store/file/eds_file.go index 5d1eefb758..fadd1883a1 100644 --- a/share/store/eds_file.go +++ b/share/store/file/eds_file.go @@ -1,4 +1,4 @@ -package store +package file import ( "context" @@ -7,6 +7,7 @@ import ( "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/shwap" ) type EdsFile interface { @@ -14,11 +15,11 @@ type EdsFile interface { // Size returns square size of the file. Size() int // Share returns share and corresponding proof for the given axis and share index in this axis. - Share(ctx context.Context, x, y int) (*share.ShareWithProof, error) - // AxisHalf returns shares for the first half of the axis of the given type and index. - AxisHalf(ctx context.Context, axisType rsmt2d.Axis, axisIdx int) ([]share.Share, error) + Share(ctx context.Context, rowIdx, colIdx int) (*shwap.Sample, error) + // AxisHalf returns Shares for the first half of the axis of the given type and index. + AxisHalf(ctx context.Context, axisType rsmt2d.Axis, axisIdx int) (AxisHalf, error) // Data returns data for the given namespace and row index. - Data(ctx context.Context, namespace share.Namespace, rowIdx int) (share.NamespacedRow, error) + Data(ctx context.Context, namespace share.Namespace, rowIdx int) (shwap.RowNamespaceData, error) // EDS returns extended data square stored in the file. EDS(ctx context.Context) (*rsmt2d.ExtendedDataSquare, error) } diff --git a/share/store/mem_file.go b/share/store/file/mem_file.go similarity index 60% rename from share/store/mem_file.go rename to share/store/file/mem_file.go index 24f9cfd110..d3fefcb29b 100644 --- a/share/store/mem_file.go +++ b/share/store/file/mem_file.go @@ -1,4 +1,4 @@ -package store +package file import ( "context" @@ -9,6 +9,7 @@ import ( "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/shwap" ) var _ EdsFile = (*MemFile)(nil) @@ -27,12 +28,12 @@ func (f *MemFile) Size() int { func (f *MemFile) Share( _ context.Context, - x, y int, -) (*share.ShareWithProof, error) { + rowIdx, colIdx int, +) (*shwap.Sample, error) { axisType := rsmt2d.Row - axisIdx, shrIdx := y, x + axisIdx, shrIdx := rowIdx, colIdx - shares := f.axis(axisType, axisIdx) + shares := getAxis(f.Eds, axisType, axisIdx) tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(f.Size()/2), uint(axisIdx)) for _, shr := range shares { err := tree.Push(shr) @@ -46,19 +47,41 @@ func (f *MemFile) Share( return nil, err } - return &share.ShareWithProof{ - Share: shares[shrIdx], - Proof: &proof, - Axis: axisType, + return &shwap.Sample{ + Share: shares[shrIdx], + Proof: &proof, + ProofType: axisType, }, nil } -func (f *MemFile) AxisHalf(_ context.Context, axisType rsmt2d.Axis, axisIdx int) ([]share.Share, error) { - return f.axis(axisType, axisIdx)[:f.Size()/2], nil +func (f *MemFile) AxisHalf(_ context.Context, axisType rsmt2d.Axis, axisIdx int) (AxisHalf, error) { + return AxisHalf{ + Shares: getAxis(f.Eds, axisType, axisIdx)[:f.Size()/2], + IsParity: false, + }, nil +} + +func (f *MemFile) Data(_ context.Context, namespace share.Namespace, rowIdx int) (shwap.RowNamespaceData, error) { + shares := getAxis(f.Eds, rsmt2d.Row, rowIdx) + return ndDataFromShares(shares, namespace, rowIdx) +} + +func (f *MemFile) EDS(_ context.Context) (*rsmt2d.ExtendedDataSquare, error) { + return f.Eds, nil } -func (f *MemFile) Data(_ context.Context, namespace share.Namespace, rowIdx int) (share.NamespacedRow, error) { - shares := f.axis(rsmt2d.Row, rowIdx) +func getAxis(eds *rsmt2d.ExtendedDataSquare, axisType rsmt2d.Axis, axisIdx int) []share.Share { + switch axisType { + case rsmt2d.Row: + return eds.Row(uint(axisIdx)) + case rsmt2d.Col: + return eds.Col(uint(axisIdx)) + default: + panic("unknown axis") + } +} + +func ndDataFromShares(shares []share.Share, namespace share.Namespace, rowIdx int) (shwap.RowNamespaceData, error) { bserv := ipld.NewMemBlockservice() batchAdder := ipld.NewNmtNodeAdder(context.TODO(), bserv, ipld.MaxSizeBatchOption(len(shares))) tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(len(shares)/2), uint(rowIdx), @@ -66,42 +89,26 @@ func (f *MemFile) Data(_ context.Context, namespace share.Namespace, rowIdx int) for _, shr := range shares { err := tree.Push(shr) if err != nil { - return share.NamespacedRow{}, err + return shwap.RowNamespaceData{}, err } } root, err := tree.Root() if err != nil { - return share.NamespacedRow{}, err + return shwap.RowNamespaceData{}, err } err = batchAdder.Commit() if err != nil { - return share.NamespacedRow{}, err + return shwap.RowNamespaceData{}, err } - cid := ipld.MustCidFromNamespacedSha256(root) - row, proof, err := ipld.GetSharesByNamespace(context.TODO(), bserv, cid, namespace, len(shares)) + row, proof, err := ipld.GetSharesByNamespace(context.TODO(), bserv, root, namespace, len(shares)) if err != nil { - return share.NamespacedRow{}, err + return shwap.RowNamespaceData{}, err } - return share.NamespacedRow{ + return shwap.RowNamespaceData{ Shares: row, Proof: proof, }, nil } - -func (f *MemFile) EDS(_ context.Context) (*rsmt2d.ExtendedDataSquare, error) { - return f.Eds, nil -} - -func (f *MemFile) axis(axisType rsmt2d.Axis, axisIdx int) []share.Share { - switch axisType { - case rsmt2d.Row: - return f.Eds.Row(uint(axisIdx)) - case rsmt2d.Col: - return f.Eds.Col(uint(axisIdx)) - default: - panic("unknown axis") - } -} diff --git a/share/store/mem_file_test.go b/share/store/file/mem_file_test.go similarity index 76% rename from share/store/mem_file_test.go rename to share/store/file/mem_file_test.go index 9df1616afd..0297dc63e6 100644 --- a/share/store/mem_file_test.go +++ b/share/store/file/mem_file_test.go @@ -1,4 +1,4 @@ -package store +package file import ( "context" @@ -19,14 +19,13 @@ func TestMemFileShare(t *testing.T) { fl := &MemFile{Eds: eds} width := int(eds.Width()) - for x := 0; x < width; x++ { - for y := 0; y < width; y++ { - shr, err := fl.Share(context.TODO(), x, y) + for rowIdx := 0; rowIdx < width; rowIdx++ { + for colIdx := 0; colIdx < width; colIdx++ { + shr, err := fl.Share(context.TODO(), rowIdx, colIdx) require.NoError(t, err) - axishash := root.RowRoots[y] - ok := shr.Validate(axishash, x, y, width) - require.True(t, ok) + err = shr.Validate(root, rowIdx, colIdx) + require.NoError(t, err) } } } @@ -45,8 +44,8 @@ func TestMemFileDate(t *testing.T) { if !namespace.IsOutsideRange(root, root) { nd, err := file.Data(context.Background(), namespace, i) require.NoError(t, err) - ok := nd.Verify(root, namespace) - require.True(t, ok) + err = nd.Validate(dah, namespace, i) + require.NoError(t, err) } } } From 7ace4abb1f40c7b3c10aa94ddca4247758e26ab7 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Wed, 22 May 2024 14:24:33 +0300 Subject: [PATCH 2/2] use common hasher --- share/shwap/row_namespace_data.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/share/shwap/row_namespace_data.go b/share/shwap/row_namespace_data.go index 177f83514a..01af228a8d 100644 --- a/share/shwap/row_namespace_data.go +++ b/share/shwap/row_namespace_data.go @@ -1,7 +1,6 @@ package shwap import ( - "crypto/sha256" "fmt" "github.com/celestiaorg/celestia-app/pkg/wrapper" @@ -158,7 +157,7 @@ func (rnd RowNamespaceData) verifyInclusion(rowRoot []byte, namespace share.Name leaves = append(leaves, append(namespaceBytes, shr...)) } return rnd.Proof.VerifyNamespace( - sha256.New(), + share.NewSHA256Hasher(), namespace.ToNMT(), leaves, rowRoot,