Merge branch 'eclesio/store-raw-authority-bytes' of github.com:ChainS…

…afe/gossamer into eclesio/store-raw-authority-bytes

dot/rpc/modules/rpc.go

@@ -19,6 +19,7 @@ var (
 		"state_getPairs",
 		"state_getKeysPaged",
 		"state_queryStorage",
+		"state_trie",
 	}
 
 	// AliasesMethods is a map that links the original methods to their aliases

dot/rpc/modules/state.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/ChainSafe/gossamer/lib/common"
 	"github.com/ChainSafe/gossamer/lib/runtime"
+	"github.com/ChainSafe/gossamer/lib/trie"
 	"github.com/ChainSafe/gossamer/pkg/scale"
 )
 
@@ -82,6 +83,10 @@ type StateStorageQueryAtRequest struct {
 	At   common.Hash `json:"at"`
 }
 
+type StateTrieAtRequest struct {
+	At *common.Hash `json:"at"`
+}
+
 // StateStorageKeysQuery field to store storage keys
 type StateStorageKeysQuery [][]byte
 
@@ -112,6 +117,8 @@ type StateStorageResponse string
 // StatePairResponse is a key values
 type StatePairResponse []interface{}
 
+type StateTrieResponse []string
+
 // StateStorageKeysResponse field for storage keys
 type StateStorageKeysResponse []string
 
@@ -245,6 +252,46 @@ func (sm *StateModule) GetPairs(_ *http.Request, req *StatePairRequest, res *Sta
 	return nil
 }
 
+// Trie RPC method returns a list of scale encoded trie.Entry{Key byte, Value byte} representing
+// all the entries in a trie for a block hash, if no block hash is given then it uses the best block hash
+func (sm *StateModule) Trie(_ *http.Request, req *StateTrieAtRequest, res *StateTrieResponse) error {
+	var blockHash common.Hash
+
+	if req.At != nil {
+		blockHash = *req.At
+	} else {
+		blockHash = sm.blockAPI.BestBlockHash()
+	}
+
+	blockHeader, err := sm.blockAPI.GetHeader(blockHash)
+	if err != nil {
+		return fmt.Errorf("getting header: %w", err)
+	}
+
+	entries, err := sm.storageAPI.Entries(&blockHeader.StateRoot)
+	if err != nil {
+		return fmt.Errorf("getting entries: %w", err)
+	}
+
+	entriesArr := make([]string, 0, len(entries))
+	for key, value := range entries {
+		entry := trie.Entry{
+			Key:   []byte(key),
+			Value: value,
+		}
+
+		encodedEntry, err := scale.Marshal(entry)
+		if err != nil {
+			return fmt.Errorf("scale encoding entry: %w", err)
+		}
+
+		entriesArr = append(entriesArr, common.BytesToHex(encodedEntry))
+	}
+
+	*res = entriesArr
+	return nil
+}
+
 // Call makes a call to the runtime.
 func (sm *StateModule) Call(_ *http.Request, req *StateCallRequest, res *StateCallResponse) error {
 	var blockHash common.Hash

dot/rpc/modules/state_test.go

@@ -18,9 +18,11 @@ package modules
 import (
 	"errors"
 	"net/http"
+	"slices"
 	"testing"
 
 	wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero"
+	"github.com/ChainSafe/gossamer/lib/trie"
 
 	"github.com/ChainSafe/gossamer/dot/rpc/modules/mocks"
 	testdata "github.com/ChainSafe/gossamer/dot/rpc/modules/test_data"
@@ -30,6 +32,7 @@ import (
 	"github.com/ChainSafe/gossamer/lib/runtime"
 	"github.com/ChainSafe/gossamer/pkg/scale"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"go.uber.org/mock/gomock"
 )
 
@@ -308,6 +311,96 @@ func TestCall(t *testing.T) {
 	assert.NotEmpty(t, res)
 }
 
+func TestStateTrie(t *testing.T) {
+	expecificBlockHash := common.Hash([32]byte{6, 6, 6, 6, 6, 6})
+	var expectedEncodedSlice []string
+	entries := []trie.Entry{
+		{Key: []byte("entry-1"), Value: []byte{0, 1, 2, 3}},
+		{Key: []byte("entry-2"), Value: []byte{3, 4, 5, 6}},
+	}
+
+	for _, entry := range entries {
+		expectedEncodedSlice = append(expectedEncodedSlice, common.BytesToHex(scale.MustMarshal(entry)))
+	}
+
+	testcases := map[string]struct {
+		request        StateTrieAtRequest
+		newStateModule func(t *testing.T) *StateModule
+		expected       StateTrieResponse
+	}{
+		"blockhash_parameter_nil": {
+			request:  StateTrieAtRequest{At: nil},
+			expected: expectedEncodedSlice,
+			newStateModule: func(t *testing.T) *StateModule {
+				ctrl := gomock.NewController(t)
+
+				bestBlockHash := common.Hash([32]byte{1, 0, 1, 0, 1})
+				blockAPIMock := NewMockBlockAPI(ctrl)
+				blockAPIMock.EXPECT().BestBlockHash().Return(bestBlockHash)
+
+				fakeStateRoot := common.Hash([32]byte{5, 5, 5, 5, 5})
+				fakeBlockHeader := types.NewHeader(common.EmptyHash, fakeStateRoot,
+					common.EmptyHash, 1, scale.VaryingDataTypeSlice{})
+
+				blockAPIMock.EXPECT().GetHeader(bestBlockHash).Return(fakeBlockHeader, nil)
+
+				fakeEntries := map[string][]byte{
+					"entry-1": {0, 1, 2, 3},
+					"entry-2": {3, 4, 5, 6},
+				}
+				storageAPIMock := NewMockStorageAPI(ctrl)
+				storageAPIMock.EXPECT().Entries(&fakeStateRoot).
+					Return(fakeEntries, nil)
+
+				sm := NewStateModule(nil, storageAPIMock, nil, blockAPIMock)
+				return sm
+			},
+		},
+		"blockhash_parameter_not_nil": {
+			request:  StateTrieAtRequest{At: &expecificBlockHash},
+			expected: expectedEncodedSlice,
+			newStateModule: func(t *testing.T) *StateModule {
+				ctrl := gomock.NewController(t)
+				blockAPIMock := NewMockBlockAPI(ctrl)
+
+				fakeStateRoot := common.Hash([32]byte{5, 5, 5, 5, 5})
+				fakeBlockHeader := types.NewHeader(common.EmptyHash, fakeStateRoot,
+					common.EmptyHash, 1, scale.VaryingDataTypeSlice{})
+
+				blockAPIMock.EXPECT().GetHeader(expecificBlockHash).
+					Return(fakeBlockHeader, nil)
+
+				fakeEntries := map[string][]byte{
+					"entry-1": {0, 1, 2, 3},
+					"entry-2": {3, 4, 5, 6},
+				}
+				storageAPIMock := NewMockStorageAPI(ctrl)
+				storageAPIMock.EXPECT().Entries(&fakeStateRoot).
+					Return(fakeEntries, nil)
+
+				sm := NewStateModule(nil, storageAPIMock, nil, blockAPIMock)
+				return sm
+			},
+		},
+	}
+
+	for tname, tt := range testcases {
+		tt := tt
+
+		t.Run(tname, func(t *testing.T) {
+			sm := tt.newStateModule(t)
+
+			var res StateTrieResponse
+			err := sm.Trie(nil, &tt.request, &res)
+			require.NoError(t, err)
+
+			slices.Sort(tt.expected)
+			slices.Sort(res)
+			require.Equal(t, tt.expected, res)
+		})
+	}
+}
+
 func TestStateModuleGetMetadata(t *testing.T) {
 	ctrl := gomock.NewController(t)
 

dot/state/epoch.go

@@ -242,7 +242,7 @@ func (s *EpochState) SetEpochDataRaw(epoch uint64, raw *types.EpochDataRaw) erro
 	return s.db.Put(epochDataKey(epoch), enc)
 }
 
-// GetEpochDataRaw returns the epoch data raw for a given epoch persisted in database
+// GetEpochDataRaw returns the raw epoch data for a given epoch persisted in database
 // otherwise will try to get the data from the in-memory map using the header
 // if the header params is nil then it will search only in database
 func (s *EpochState) GetEpochDataRaw(epoch uint64, header *types.Header) (*types.EpochDataRaw, error) {

go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/tetratelabs/wazero v1.1.0
 	github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9
 	go.uber.org/mock v0.3.0
-	golang.org/x/crypto v0.15.0
+	golang.org/x/crypto v0.16.0
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
 	golang.org/x/term v0.15.0
 	google.golang.org/protobuf v1.31.0
@@ -203,6 +203,6 @@ require (
 
 go 1.21
 
-replace github.com/tetratelabs/wazero => github.com/ChainSafe/wazero v0.0.0-20230710171859-39a4c235ec1f
+replace github.com/tetratelabs/wazero => github.com/ChainSafe/wazero v0.0.0-20231114190045-1d874d099362
 
 replace github.com/centrifuge/go-substrate-rpc-client/v4 => github.com/timwu20/go-substrate-rpc-client/v4 v4.0.0-20231110032757-3d8e441b7303

go.sum

@@ -48,8 +48,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/ChainSafe/go-schnorrkel v1.1.0 h1:rZ6EU+CZFCjB4sHUE1jIu8VDoB/wRKZxoe1tkcO71Wk=
 github.com/ChainSafe/go-schnorrkel v1.1.0/go.mod h1:ABkENxiP+cvjFiByMIZ9LYbRoNNLeBLiakC1XeTFxfE=
-github.com/ChainSafe/wazero v0.0.0-20230710171859-39a4c235ec1f h1:/sI8TMJ77HL2UImQs7pY7khVN96EXQJGVOrX88dTpcY=
-github.com/ChainSafe/wazero v0.0.0-20230710171859-39a4c235ec1f/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ=
+github.com/ChainSafe/wazero v0.0.0-20231114190045-1d874d099362 h1:hbvvSSB436JJalwq/2fRZwJpptvq9HMOLYVZX9oVHKM=
+github.com/ChainSafe/wazero v0.0.0-20231114190045-1d874d099362/go.mod h1:0U0G41+ochRKoPKCJlh0jMg1CHkyfK8kDqiirMmKY8A=
 github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno=
 github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo=
 github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
@@ -868,8 +868,8 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
-golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
+golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
+golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=

lib/runtime/allocator/freeing_bump.go

@@ -37,7 +37,7 @@ const (
 	MaxPossibleAllocations uint32 = 33554432
 
 	PageSize     = 65536
-	MaxWasmPages = 4 * 1024 * 1024 * 1024 / PageSize
+	MaxWasmPages = (4 * 1024 * 1024 * 1024 / PageSize) - 1
 )
 
 var (

lib/runtime/memory.go

@@ -3,9 +3,6 @@
 
 package runtime
 
-// PageSize is 65kb
-const PageSize = 65536
-
 // Memory is the interface for WASM memory
 type Memory interface {
 	// Size returns the size in bytes available. e.g. If the underlying memory

tests/rpc/rpc_05-state_test.go

@@ -10,10 +10,11 @@ import (
 	"time"
 
 	"github.com/ChainSafe/gossamer/dot/rpc/modules"
-	"github.com/ChainSafe/gossamer/lib/runtime"
-
 	"github.com/ChainSafe/gossamer/lib/common"
+	"github.com/ChainSafe/gossamer/lib/runtime"
+	"github.com/ChainSafe/gossamer/lib/trie"
 	libutils "github.com/ChainSafe/gossamer/lib/utils"
+	"github.com/ChainSafe/gossamer/pkg/scale"
 	"github.com/ChainSafe/gossamer/tests/utils/config"
 	"github.com/ChainSafe/gossamer/tests/utils/node"
 	"github.com/ChainSafe/gossamer/tests/utils/rpc"
@@ -33,6 +34,32 @@ func TestStateRPCResponseValidation(t *testing.T) { //nolint:tparallel
 	getBlockHashCancel()
 	require.NoError(t, err)
 
+	t.Run("state_trie", func(t *testing.T) {
+		t.Parallel()
+		const westendDevGenesisHash = "0x276bfa91f70859348285599321ea96afd3ae681f0be47d36196bac8075ea32e8"
+		const westendDevStateRoot = "0x953044ba4386a72ae434d2a2fbdfca77640a28ac3841a924674cbfe7a8b9a81c"
+		params := fmt.Sprintf(`["%s"]`, westendDevGenesisHash)
+
+		var response modules.StateTrieResponse
+		fetchWithTimeout(ctx, t, "state_trie", params, &response)
+
+		entries := make(map[string]string, len(response))
+		for _, encodedEntry := range response {
+			bytesEncodedEntry := common.MustHexToBytes(encodedEntry)
+
+			entry := trie.Entry{}
+			err := scale.Unmarshal(bytesEncodedEntry, &entry)
+			require.NoError(t, err)
+			entries[common.BytesToHex(entry.Key)] = common.BytesToHex(entry.Value)
+		}
+
+		newTrie, err := trie.LoadFromMap(entries)
+		require.NoError(t, err)
+
+		trieHash := newTrie.MustHash(trie.V0.MaxInlineValue())
+		require.Equal(t, westendDevStateRoot, trieHash.String())
+	})
+
 	// TODO: Improve runtime tests
 	// https://github.com/ChainSafe/gossamer/issues/3234
 	t.Run("state_call", func(t *testing.T) {