chore(lib/trie): refactor encoding and hash related code in trie pack…

…age (#2077)

* Chore(packages): move node interface, leaf & branch implementations, encoding and decoding functions in `internal/trie/node`
* Chore(packages): create `internal/trie/recorder` subpackage (#2082) 
* Chore(packages): create `internal/trie/pools` subpackage
* Chore(packages): create `internal/trie/codec` subpackage
* Chore(tests): add tests with near full coverage
* Chore(errors): improve error wrapping on trie node implementations and encoding/decoding
* Optimization: use `sync.Pool` for header byte reading
* Optimization: encode headers directly to buffer
* Code addition: `GetValue() []byte` method for node interface
* Code addition: `GetKey() []byte` method for node interface
* Chore(comments): add and clarify existing comments
* Chore(api): unexport node implementation fields: `Generation`, `Dirty`, `Encoding` and `Hash`
* Minor change: trie `string()` method does not cache encoding in nodes. This is only used for debugging.

Co-authored-by: Kishan Sagathiya <kishansagathiya@gmail.com>

internal/trie/codec/nibbles.go

@@ -0,0 +1,48 @@
+// Copyright 2021 ChainSafe Systems (ON)
+// SPDX-License-Identifier: LGPL-3.0-only
+
+package codec
+
+// NibblesToKeyLE converts a slice of nibbles with length k into a
+// Little Endian byte slice.
+// It assumes nibbles are already in Little Endian and does not rearrange nibbles.
+// If the length of the input is odd, the result is
+// [ 0000 in[0] | in[1] in[2] | ... | in[k-2] in[k-1] ]
+// Otherwise, the result is
+// [ in[0] in[1] | ... | in[k-2] in[k-1] ]
+func NibblesToKeyLE(nibbles []byte) []byte {
+	if len(nibbles)%2 == 0 {
+		keyLE := make([]byte, len(nibbles)/2)
+		for i := 0; i < len(nibbles); i += 2 {
+			keyLE[i/2] = (nibbles[i] << 4 & 0xf0) | (nibbles[i+1] & 0xf)
+		}
+		return keyLE
+	}
+
+	keyLE := make([]byte, len(nibbles)/2+1)
+	keyLE[0] = nibbles[0]
+	for i := 2; i < len(nibbles); i += 2 {
+		keyLE[i/2] = (nibbles[i-1] << 4 & 0xf0) | (nibbles[i] & 0xf)
+	}
+
+	return keyLE
+}
+
+// KeyLEToNibbles converts a Little Endian byte slice into nibbles.
+// It assumes bytes are already in Little Endian and does not rearrange nibbles.
+func KeyLEToNibbles(in []byte) (nibbles []byte) {
+	if len(in) == 0 {
+		return []byte{}
+	} else if len(in) == 1 && in[0] == 0 {
+		return []byte{0, 0}
+	}
+
+	l := len(in) * 2
+	nibbles = make([]byte, l)
+	for i, b := range in {
+		nibbles[2*i] = b / 16
+		nibbles[2*i+1] = b % 16
+	}
+
+	return nibbles
+}

internal/trie/codec/nibbles_test.go

@@ -0,0 +1,142 @@
+// Copyright 2021 ChainSafe Systems (ON)
+// SPDX-License-Identifier: LGPL-3.0-only
+
+package codec
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_NibblesToKeyLE(t *testing.T) {
+	t.Parallel()
+
+	testCases := map[string]struct {
+		nibbles []byte
+		keyLE   []byte
+	}{
+		"nil nibbles": {
+			keyLE: []byte{},
+		},
+		"empty nibbles": {
+			nibbles: []byte{},
+			keyLE:   []byte{},
+		},
+		"0xF 0xF": {
+			nibbles: []byte{0xF, 0xF},
+			keyLE:   []byte{0xFF},
+		},
+		"0x3 0xa 0x0 0x5": {
+			nibbles: []byte{0x3, 0xa, 0x0, 0x5},
+			keyLE:   []byte{0x3a, 0x05},
+		},
+		"0xa 0xa 0xf 0xf 0x0 0x1": {
+			nibbles: []byte{0xa, 0xa, 0xf, 0xf, 0x0, 0x1},
+			keyLE:   []byte{0xaa, 0xff, 0x01},
+		},
+		"0xa 0xa 0xf 0xf 0x0 0x1 0xc 0x2": {
+			nibbles: []byte{0xa, 0xa, 0xf, 0xf, 0x0, 0x1, 0xc, 0x2},
+			keyLE:   []byte{0xaa, 0xff, 0x01, 0xc2},
+		},
+		"0xa 0xa 0xf 0xf 0x0 0x1 0xc": {
+			nibbles: []byte{0xa, 0xa, 0xf, 0xf, 0x0, 0x1, 0xc},
+			keyLE:   []byte{0xa, 0xaf, 0xf0, 0x1c},
+		},
+	}
+
+	for name, testCase := range testCases {
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			keyLE := NibblesToKeyLE(testCase.nibbles)
+
+			assert.Equal(t, testCase.keyLE, keyLE)
+		})
+	}
+}
+
+func Test_KeyLEToNibbles(t *testing.T) {
+	t.Parallel()
+
+	testCases := map[string]struct {
+		in      []byte
+		nibbles []byte
+	}{
+		"nil input": {
+			nibbles: []byte{},
+		},
+		"empty input": {
+			in:      []byte{},
+			nibbles: []byte{},
+		},
+		"0x0": {
+			in:      []byte{0x0},
+			nibbles: []byte{0, 0}},
+		"0xFF": {
+			in:      []byte{0xFF},
+			nibbles: []byte{0xF, 0xF}},
+		"0x3a 0x05": {
+			in:      []byte{0x3a, 0x05},
+			nibbles: []byte{0x3, 0xa, 0x0, 0x5}},
+		"0xAA 0xFF 0x01": {
+			in:      []byte{0xAA, 0xFF, 0x01},
+			nibbles: []byte{0xa, 0xa, 0xf, 0xf, 0x0, 0x1}},
+		"0xAA 0xFF 0x01 0xc2": {
+			in:      []byte{0xAA, 0xFF, 0x01, 0xc2},
+			nibbles: []byte{0xa, 0xa, 0xf, 0xf, 0x0, 0x1, 0xc, 0x2}},
+		"0xAA 0xFF 0x01 0xc0": {
+			in:      []byte{0xAA, 0xFF, 0x01, 0xc0},
+			nibbles: []byte{0xa, 0xa, 0xf, 0xf, 0x0, 0x1, 0xc, 0x0}},
+	}
+
+	for name, testCase := range testCases {
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			nibbles := KeyLEToNibbles(testCase.in)
+
+			assert.Equal(t, testCase.nibbles, nibbles)
+		})
+	}
+}
+
+func Test_NibblesKeyLE(t *testing.T) {
+	t.Parallel()
+
+	testCases := map[string]struct {
+		nibblesToEncode []byte
+		nibblesDecoded  []byte
+	}{
+		"empty input": {
+			nibblesToEncode: []byte{},
+			nibblesDecoded:  []byte{},
+		},
+		"one byte": {
+			nibblesToEncode: []byte{1},
+			nibblesDecoded:  []byte{0, 1},
+		},
+		"two bytes": {
+			nibblesToEncode: []byte{1, 2},
+			nibblesDecoded:  []byte{1, 2},
+		},
+		"three bytes": {
+			nibblesToEncode: []byte{1, 2, 3},
+			nibblesDecoded:  []byte{0, 1, 2, 3},
+		},
+	}
+
+	for name, testCase := range testCases {
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			keyLE := NibblesToKeyLE(testCase.nibblesToEncode)
+			nibblesDecoded := KeyLEToNibbles(keyLE)
+
+			assert.Equal(t, testCase.nibblesDecoded, nibblesDecoded)
+		})
+	}
+}

internal/trie/node/branch.go

@@ -0,0 +1,50 @@
+// Copyright 2021 ChainSafe Systems (ON)
+// SPDX-License-Identifier: LGPL-3.0-only
+
+package node
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/ChainSafe/gossamer/lib/common"
+)
+
+var _ Node = (*Branch)(nil)
+
+// Branch is a branch in the trie.
+type Branch struct {
+	Key      []byte // partial key
+	Children [16]Node
+	Value    []byte
+	// dirty is true when the branch differs
+	// from the node stored in the database.
+	dirty      bool
+	hashDigest []byte
+	encoding   []byte
+	// generation is incremented on every trie Snapshot() call.
+	// Each node also contain a certain generation number,
+	// which is updated to match the trie generation once they are
+	// inserted, moved or iterated over.
+	generation uint64
+	sync.RWMutex
+}
+
+// NewBranch creates a new branch using the arguments given.
+func NewBranch(key, value []byte, dirty bool, generation uint64) *Branch {
+	return &Branch{
+		Key:        key,
+		Value:      value,
+		dirty:      dirty,
+		generation: generation,
+	}
+}
+
+func (b *Branch) String() string {
+	if len(b.Value) > 1024 {
+		return fmt.Sprintf("branch key=0x%x childrenBitmap=%b value (hashed)=0x%x dirty=%t",
+			b.Key, b.ChildrenBitmap(), common.MustBlake2bHash(b.Value), b.dirty)
+	}
+	return fmt.Sprintf("branch key=0x%x childrenBitmap=%b value=0x%x dirty=%t",
+		b.Key, b.ChildrenBitmap(), b.Value, b.dirty)
+}