introduce File interface and decouple ipldv2 tests from on disk file

share/eds/file.go

@@ -14,6 +14,15 @@ import (
 	"github.com/celestiaorg/celestia-node/share"
 )
 
+type File interface {
+	io.Closer
+	Size() int
+	ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof, error)
+	Axis(idx int, axis rsmt2d.Axis) ([]share.Share, error)
+	AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error)
+	EDS() (*rsmt2d.ExtendedDataSquare, error)
+}
+
 type FileConfig struct {
 	Version     FileVersion
 	Compression FileCompression
@@ -23,13 +32,13 @@ type FileConfig struct {
 	// TODO: Add codec
 }
 
-// File
+// LazyFile
 // * immutable
 // * versionable
 // TODO:
 //   - Cache Rows and Cols
 //   - Avoid storing constant shares, like padding
-type File struct {
+type LazyFile struct {
 	path string
 	hdr  *Header
 	fl   fileBackend
@@ -40,7 +49,7 @@ type fileBackend interface {
 	io.Closer
 }
 
-func OpenFile(path string) (*File, error) {
+func OpenFile(path string) (*LazyFile, error) {
 	f, err := mmap.Open(path)
 	if err != nil {
 		return nil, err
@@ -52,14 +61,14 @@ func OpenFile(path string) (*File, error) {
 	}
 
 	// TODO(WWondertan): Validate header
-	return &File{
+	return &LazyFile{
 		path: path,
 		hdr:  h,
 		fl:   f,
 	}, nil
 }
 
-func CreateFile(path string, eds *rsmt2d.ExtendedDataSquare, cfgs ...FileConfig) (*File, error) {
+func CreateFile(path string, eds *rsmt2d.ExtendedDataSquare, cfgs ...FileConfig) (*LazyFile, error) {
 	f, err := os.Create(path)
 	if err != nil {
 		return nil, err
@@ -94,22 +103,26 @@ func CreateFile(path string, eds *rsmt2d.ExtendedDataSquare, cfgs ...FileConfig)
 		}
 	}
 
-	return &File{
+	return &LazyFile{
 		path: path,
 		fl:   f,
 		hdr:  h,
 	}, f.Sync()
 }
 
-func (f *File) Close() error {
+func (f *LazyFile) Size() int {
+	return f.hdr.SquareSize()
+}
+
+func (f *LazyFile) Close() error {
 	return f.fl.Close()
 }
 
-func (f *File) Header() *Header {
+func (f *LazyFile) Header() *Header {
 	return f.hdr
 }
 
-func (f *File) Axis(idx int, axis rsmt2d.Axis) ([]share.Share, error) {
+func (f *LazyFile) Axis(idx int, axis rsmt2d.Axis) ([]share.Share, error) {
 	shrLn := int(f.hdr.shareSize)
 	sqrLn := int(f.hdr.squareSize)
 	if f.Header().Config().Mode == ODSMode {
@@ -156,7 +169,7 @@ func (f *File) Axis(idx int, axis rsmt2d.Axis) ([]share.Share, error) {
 	return shrs, nil
 }
 
-func (f *File) AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error) {
+func (f *LazyFile) AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error) {
 	// TODO(@Wondertan): this has to read directly from the file, avoiding recompute
 	fullAxis, err := f.Axis(idx, axis)
 	if err != nil {
@@ -166,19 +179,7 @@ func (f *File) AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error) {
 	return fullAxis[:len(fullAxis)/2], nil
 }
 
-func (f *File) Share(idx int) (share.Share, error) {
-	// TODO: Check the cache first
-	shrLn := int64(f.hdr.shareSize)
-
-	offset := int64(idx)*shrLn + HeaderSize
-	shr := make(share.Share, shrLn)
-	if _, err := f.fl.ReadAt(shr, offset); err != nil {
-		return nil, err
-	}
-	return shr, nil
-}
-
-func (f *File) ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof, error) {
+func (f *LazyFile) ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof, error) {
 	// TODO: Cache the axis as well as computed tree
 	sqrLn := int(f.hdr.squareSize)
 	axsIdx, shrIdx := idx/sqrLn, idx%sqrLn
@@ -207,7 +208,7 @@ func (f *File) ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof
 	return shrs[shrIdx], proof, nil
 }
 
-func (f *File) EDS() (*rsmt2d.ExtendedDataSquare, error) {
+func (f *LazyFile) EDS() (*rsmt2d.ExtendedDataSquare, error) {
 	shrLn := int(f.hdr.shareSize)
 	sqrLn := int(f.hdr.squareSize)
 	if f.Header().Config().Mode == ODSMode {

share/eds/file_store.go

@@ -6,7 +6,7 @@ type FileStore struct {
 	baspath string
 }
 
-func (fs *FileStore) File(hash share.DataHash) (*File, error) {
+func (fs *FileStore) File(hash share.DataHash) (File, error) {
 	// TODO(@Wondertan): Caching
 	return OpenFile(fs.baspath + "/" + hash.String())
 }

share/eds/file_test.go

@@ -54,10 +54,6 @@ func TestFile(t *testing.T) {
 	for _, axis := range axis {
 		for i := 0; i < width*width; i++ {
 			row, col := uint(i/width), uint(i%width)
-			shr, err := fl.Share(i)
-			require.NoError(t, err)
-			assert.EqualValues(t, eds.GetCell(row, col), shr)
-
 			shr, prf, err := fl.ShareWithProof(i, axis)
 			require.NoError(t, err)
 			assert.EqualValues(t, eds.GetCell(row, col), shr)
@@ -84,15 +80,3 @@ func TestFile(t *testing.T) {
 	err = fl.Close()
 	require.NoError(t, err)
 }
-
-// TODO(@Wondertan): Should be a method on eds
-func getAxis(idx int, axis rsmt2d.Axis, eds *rsmt2d.ExtendedDataSquare) [][]byte {
-	switch axis {
-	case rsmt2d.Row:
-		return eds.Row(uint(idx))
-	case rsmt2d.Col:
-		return eds.Col(uint(idx))
-	default:
-		panic("")
-	}
-}

share/eds/ods_file.go

@@ -0,0 +1,74 @@
+package eds
+
+import (
+	"github.com/celestiaorg/celestia-app/pkg/wrapper"
+	"github.com/celestiaorg/nmt"
+	"github.com/celestiaorg/rsmt2d"
+
+	"github.com/celestiaorg/celestia-node/share"
+)
+
+type MemFile struct {
+	Eds *rsmt2d.ExtendedDataSquare
+}
+
+func (f *MemFile) Close() error {
+	return nil
+}
+
+func (f *MemFile) Size() int {
+	return int(f.Eds.Width())
+}
+
+func (f *MemFile) ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof, error) {
+	sqrLn := f.Size()
+	axsIdx, shrIdx := idx/sqrLn, idx%sqrLn
+	if axis == rsmt2d.Col {
+		axsIdx, shrIdx = shrIdx, axsIdx
+	}
+
+	shrs, err := f.Axis(axsIdx, axis)
+	if err != nil {
+		return nil, nmt.Proof{}, err
+	}
+
+	// TODO(@Wondartan): this must access cached NMT on EDS instead of computing a new one
+	tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(sqrLn/2), uint(axsIdx))
+	for _, shr := range shrs {
+		err = tree.Push(shr)
+		if err != nil {
+			return nil, nmt.Proof{}, err
+		}
+	}
+
+	proof, err := tree.ProveRange(shrIdx, shrIdx+1)
+	if err != nil {
+		return nil, nmt.Proof{}, err
+	}
+
+	return shrs[shrIdx], proof, nil
+}
+
+func (f *MemFile) Axis(idx int, axis rsmt2d.Axis) ([]share.Share, error) {
+	return getAxis(idx, axis, f.Eds), nil
+}
+
+func (f *MemFile) AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error) {
+	return getAxis(idx, axis, f.Eds)[:f.Size()/2], nil
+}
+
+func (f *MemFile) EDS() (*rsmt2d.ExtendedDataSquare, error) {
+	return f.Eds, nil
+}
+
+// TODO(@Wondertan): Should be a method on eds
+func getAxis(idx int, axis rsmt2d.Axis, eds *rsmt2d.ExtendedDataSquare) [][]byte {
+	switch axis {
+	case rsmt2d.Row:
+		return eds.Row(uint(idx))
+	case rsmt2d.Col:
+		return eds.Col(uint(idx))
+	default:
+		panic("unknown axis")
+	}
+}

share/ipldv2/axis_sample.go

@@ -134,7 +134,7 @@ func (s *AxisSample) Validate() error {
 	}
 	s.AxisHalf = append(s.AxisHalf, parity...)
 
-	tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(len(s.AxisHalf)), uint(s.ID.Index))
+	tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(len(s.AxisHalf)/2), uint(s.ID.Index))
 	for _, shr := range s.AxisHalf {
 		err := tree.Push(shr)
 		if err != nil {

share/ipldv2/blockstore.go

@@ -3,38 +3,26 @@ package ipldv2
 import (
 	"context"
 	"fmt"
-	"io"
 
 	"github.com/ipfs/boxo/blockstore"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 
-	"github.com/celestiaorg/nmt"
-	"github.com/celestiaorg/rsmt2d"
-
 	"github.com/celestiaorg/celestia-node/share"
+	"github.com/celestiaorg/celestia-node/share/eds"
 )
 
-// edsFile is a mocking friendly local interface over eds.File.
-// TODO(@Wondertan): Consider making an actual interface of eds pkg
-type edsFile interface {
-	io.Closer
-	Size() int
-	ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof, error)
-	AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error)
-}
-
 // fileStore is a mocking friendly local interface over eds.FileStore
 // TODO(@Wondertan): Consider making an actual interface of eds pkg
-type fileStore[F edsFile] interface {
+type fileStore[F eds.File] interface {
 	File(share.DataHash) (F, error)
 }
 
-type Blockstore[F edsFile] struct {
+type Blockstore[F eds.File] struct {
 	fs fileStore[F]
 }
 
-func NewBlockstore[F edsFile](fs fileStore[F]) blockstore.Blockstore {
+func NewBlockstore[F eds.File](fs fileStore[F]) blockstore.Blockstore {
 	return &Blockstore[F]{fs}
 }
 

share/ipldv2/blockstore_test.go

@@ -4,27 +4,26 @@ import (
 	"context"
 	"testing"
 
+	"github.com/ipfs/boxo/blockstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/celestiaorg/nmt"
 	"github.com/celestiaorg/rsmt2d"
 
 	"github.com/celestiaorg/celestia-node/share"
 	"github.com/celestiaorg/celestia-node/share/eds"
 	"github.com/celestiaorg/celestia-node/share/eds/edstest"
 )
 
-func TestBlockstoreGet(t *testing.T) {
+// TODO(@Wondertan): Add axis sampling code
+
+func TestBlockstoreGetShareSample(t *testing.T) {
 	ctx := context.Background()
 	sqr := edstest.RandEDS(t, 4)
 	root, err := share.NewRoot(sqr)
 	require.NoError(t, err)
 
-	path := t.TempDir() + "/eds_file"
-	f, err := eds.CreateFile(path, sqr)
-	require.NoError(t, err)
-	b := NewBlockstore[*edsFileAndFS]((*edsFileAndFS)(f))
+	b := edsBlockstore(sqr)
 
 	axis := []rsmt2d.Axis{rsmt2d.Row, rsmt2d.Col}
 	width := int(sqr.Width())
@@ -47,24 +46,12 @@ func TestBlockstoreGet(t *testing.T) {
 	}
 }
 
-type edsFileAndFS eds.File
-
-func (m *edsFileAndFS) File(share.DataHash) (*edsFileAndFS, error) {
-	return m, nil
-}
-
-func (m *edsFileAndFS) Size() int {
-	return (*eds.File)(m).Header().SquareSize()
-}
-
-func (m *edsFileAndFS) ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof, error) {
-	return (*eds.File)(m).ShareWithProof(idx, axis)
-}
+type edsFileAndFS eds.MemFile
 
-func (m *edsFileAndFS) AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error) {
-	return (*eds.File)(m).AxisHalf(idx, axis)
+func (m *edsFileAndFS) File(share.DataHash) (*eds.MemFile, error) {
+	return (*eds.MemFile)(m), nil
 }
 
-func (m *edsFileAndFS) Close() error {
-	return nil
+func edsBlockstore(sqr *rsmt2d.ExtendedDataSquare) blockstore.Blockstore {
+	return NewBlockstore[*eds.MemFile]((*edsFileAndFS)(&eds.MemFile{Eds: sqr}))
 }