Skip to content

Commit

Permalink
cmd: add hash trie node prune tool
Browse files Browse the repository at this point in the history
  • Loading branch information
@fynnss
fynnss committed Sep 25, 2023
1 parent 5dc92c9 commit f60e666
Show file tree
Hide file tree
Showing 4 changed files with 247 additions and 43 deletions.
200 changes: 165 additions & 35 deletions cmd/geth/dbcmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ Remove blockchain and state databases`,
// dbMigrateFreezerCmd,
dbCheckStateContentCmd,
dbHash2PathCmd,
dbPruneHashTrieCmd,
dbTrieGetCmd,
},
}
Expand Down Expand Up @@ -117,9 +118,21 @@ a data corruption.`,
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.StateSchemeFlag,
},
Description: "This command looks up the specified trie node key from the database.",
}
dbPruneHashTrieCmd = &cli.Command{
Action: pruneHashTrie,
Name: "prune-hash-trie",
ArgsUsage: "",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
},
Usage: "[Caution]Prune all the hash trie node in diskdb",
Description: `This command iterates the entrie kv in leveldb and delete all the hash trie node.`,
}
dbStatCmd = &cli.Command{
Action: dbStats,
Name: "stats",
Expand Down Expand Up @@ -151,6 +164,19 @@ corruption if it is aborted during execution'!`,
}, utils.NetworkFlags, utils.DatabasePathFlags),
Description: "This command looks up the specified database key from the database.",
}
dbTrieDeleteCmd = &cli.Command{
Action: dbTrieDelete,
Name: "trieget",
Usage: "delete the specify trie node",
ArgsUsage: "[trie owner] <hash-base key> | <path-base key>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.StateSchemeFlag,
},
Description: "This command looks up the specified trie node key from the database.",
}
dbDeleteCmd = &cli.Command{
Action: dbDelete,
Name: "delete",
Expand Down Expand Up @@ -295,37 +321,6 @@ func confirmAndRemoveDB(database string, kind string) {
}
}

func inspect(ctx *cli.Context) error {
var (
prefix []byte
start []byte
)
if ctx.NArg() > 2 {
return fmt.Errorf("max 2 arguments: %v", ctx.Command.ArgsUsage)
}
if ctx.NArg() >= 1 {
if d, err := hexutil.Decode(ctx.Args().Get(0)); err != nil {
return fmt.Errorf("failed to hex-decode 'prefix': %v", err)
} else {
prefix = d
}
}
if ctx.NArg() >= 2 {
if d, err := hexutil.Decode(ctx.Args().Get(1)); err != nil {
return fmt.Errorf("failed to hex-decode 'start': %v", err)
} else {
start = d
}
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()

db := utils.MakeChainDatabase(ctx, stack, true, false)
defer db.Close()

return rawdb.InspectDatabase(db, prefix, start)
}

func ancientInspect(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
Expand Down Expand Up @@ -395,17 +390,26 @@ func hash2Path(ctx *cli.Context) error {
return nil
}

// dbTrieGet shows the value of a given database key
func dbTrieGet(ctx *cli.Context) error {
if ctx.NArg() < 1 || ctx.NArg() > 2 {
return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
func pruneHashTrie(ctx *cli.Context) error {
if ctx.NArg() != 0 {
return fmt.Errorf("required none argument")
}

stack, _ := makeConfigNode(ctx)
defer stack.Close()

db := utils.MakeChainDatabase(ctx, stack, false, false)
defer db.Close()

return rawdb.PruneHashTrieNodeInDataBase(db)
}

func inspect(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

db := utils.MakeChainDatabase(ctx, stack, false, false)
defer db.Close()
var (
pathKey []byte
owner []byte
Expand Down Expand Up @@ -562,6 +566,132 @@ func dbGet(ctx *cli.Context) error {
return nil
}

// dbTrieGet shows the value of a given database key
func dbTrieGet(ctx *cli.Context) error {
if ctx.NArg() < 1 || ctx.NArg() > 2 {
return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()

db := utils.MakeChainDatabase(ctx, stack, false, false)
defer db.Close()

scheme := ctx.String(utils.StateSchemeFlag.Name)
if scheme == "" {
scheme = rawdb.HashScheme
}

if scheme == rawdb.PathScheme {
var (
pathKey []byte
owner []byte
err error
)
if ctx.NArg() == 1 {
pathKey, err = hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}
nodeVal, hash := rawdb.ReadAccountTrieNode(db, pathKey)
fmt.Println("PathKey: ", common.Bytes2Hex(pathKey), "Hash: ", hash, "node: ", trie.NodeString(hash.Bytes(), nodeVal))
} else if ctx.NArg() == 2 {
owner, err = hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}
pathKey, err = hexutil.Decode(ctx.Args().Get(1))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}

nodeVal, hash := rawdb.ReadStorageTrieNode(db, common.BytesToHash(owner), pathKey)
fmt.Println("PathKey: ", common.Bytes2Hex(pathKey), "Owner: ", common.BytesToHash(owner), "Hash: ", hash, "node: ", trie.NodeString(hash.Bytes(), nodeVal))
}
} else if scheme == rawdb.HashScheme {
if ctx.NArg() == 1 {
hashKey, err := hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}
val, err := db.Get(hashKey)
if err != nil {
fmt.Println("db get error: ", err)
return err
}
fmt.Println("HashKey: ", common.BytesToHash(hashKey), "node: ", trie.NodeString(hashKey, val))
} else {
fmt.Println("args too much")
}
}

return nil
}

// dbTrieDelete delete the trienode of a given database key
func dbTrieDelete(ctx *cli.Context) error {
if ctx.NArg() < 1 || ctx.NArg() > 2 {
return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()

db := utils.MakeChainDatabase(ctx, stack, false, false)
defer db.Close()

scheme := ctx.String(utils.StateSchemeFlag.Name)
if scheme == "" {
scheme = rawdb.HashScheme
}

if scheme == rawdb.PathScheme {
var (
pathKey []byte
owner []byte
err error
)
if ctx.NArg() == 1 {
pathKey, err = hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}
rawdb.DeleteAccountTrieNode(db, pathKey)
} else if ctx.NArg() == 2 {
owner, err = hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}
pathKey, err = hexutil.Decode(ctx.Args().Get(1))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}
rawdb.DeleteStorageTrieNode(db, common.BytesToHash(owner), pathKey)
}
} else if scheme == rawdb.HashScheme {
if ctx.NArg() == 1 {
hashKey, err := hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}
err = db.Delete(hashKey)
if err != nil {
fmt.Println("db delete failed, err: ", err)
}
} else {
fmt.Println("args too much")
}
}
return nil
}

// dbDelete deletes a key from the database
func dbDelete(ctx *cli.Context) error {
if ctx.NArg() != 1 {
Expand Down
22 changes: 22 additions & 0 deletions core/rawdb/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -581,6 +581,28 @@ func AncientInspect(db ethdb.Database) error {
return nil
}

func PruneHashTrieNodeInDataBase(db ethdb.Database) error {
it := db.NewIterator([]byte{}, []byte{})
defer it.Release()

total_num := 0
for it.Next() {
var key = it.Key()
switch {
case IsLegacyTrieNode(key, it.Value()):
db.Delete(key)
total_num++
if total_num%100000 == 0 {
fmt.Printf("Complete progress: %v hash-base trie nodes\n", total_num)
}
default:
continue
}
}
fmt.Printf("Complete progress: %v hash-base trie nodes\n", total_num)
return nil
}

// InspectDatabase traverses the entire database and checks the size
// of all different categories of data.
func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
Expand Down
63 changes: 55 additions & 8 deletions trie/hash2path.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,8 @@ func (h2p *Hash2Path) Run() {
h2p.ConcurrentTraversal(h2p.trie, h2p.root, []byte{})
h2p.wg.Wait()

fmt.Println("hash2Path run finished.")
fmt.Printf("Total complete: %v, go routines Num: %v, h2p concurrentQueue: %v\n", h2p.totalNum, runtime.NumGoroutine(), len(h2p.concurrentQueue))

rawdb.WritePersistentStateID(h2p.db.diskdb, h2p.blocknum)
rawdb.WriteStateID(h2p.db.diskdb, h2p.stateRootHash, h2p.blocknum)
}
Expand All @@ -94,15 +95,11 @@ func (h2p *Hash2Path) SubConcurrentTraversal(theTrie *Trie, theNode node, path [
h2p.ConcurrentTraversal(theTrie, theNode, path)
<-h2p.concurrentQueue
h2p.wg.Done()

}

func (h2p *Hash2Path) ConcurrentTraversal(theTrie *Trie, theNode node, path []byte) {
// print process progress
total_num := atomic.AddUint64(&h2p.totalNum, 1)
if total_num%100000 == 0 {
fmt.Printf("Complete progress: %v, go routines Num: %v, h2p concurrentQueue: %v\n", total_num, runtime.NumGoroutine(), len(h2p.concurrentQueue))
}

total_num := uint64(0)
// nil node
if theNode == nil {
return
Expand All @@ -116,11 +113,19 @@ func (h2p *Hash2Path) ConcurrentTraversal(theTrie *Trie, theNode node, path []by
h2p.writeNode(path, trienode.New(common.BytesToHash(hash), nodeToBytes(collapsed)), theTrie.owner)

h2p.ConcurrentTraversal(theTrie, current.Val, append(path, current.Key...))

case *fullNode:
// copy from trie/Committer (*committer).commit
collapsed := current.copy()
collapsed.Children = h2p.commitChildren(path, current)
var hash, _ = collapsed.cache()
collapsed.Children = h2p.commitChildren(path, current)

nodebytes := nodeToBytes(collapsed)
if common.BytesToHash(hash) != common.BytesToHash(crypto.Keccak256(nodebytes)) {
fmt.Println("Hash is inconsistent, hash: ", common.BytesToHash(hash), "node hash: ", common.BytesToHash(crypto.Keccak256(nodebytes)), "node: ", collapsed.fstring(""))
panic("hash inconsistent.")
}

h2p.writeNode(path, trienode.New(common.BytesToHash(hash), nodeToBytes(collapsed)), theTrie.owner)

for idx, child := range current.Children {
Expand All @@ -144,6 +149,10 @@ func (h2p *Hash2Path) ConcurrentTraversal(theTrie *Trie, theNode node, path []by
return
}
h2p.ConcurrentTraversal(theTrie, n, path)
total_num = atomic.AddUint64(&h2p.totalNum, 1)
if total_num%100000 == 0 {
fmt.Printf("Complete progress: %v, go routines Num: %v, h2p concurrentQueue: %v\n", total_num, runtime.NumGoroutine(), len(h2p.concurrentQueue))
}
return
case valueNode:
if !hasTerm(path) {
Expand Down Expand Up @@ -189,10 +198,48 @@ func (h2p *Hash2Path) commitChildren(path []byte, n *fullNode) [17]node {
children[i] = hn
continue
}

children[i] = h2p.commit(append(path, byte(i)), child)
}
// For the 17th child, it's possible the type is valuenode.
if n.Children[16] != nil {
children[16] = n.Children[16]
}
return children
}

// commit collapses a node down into a hash node and returns it.
func (h2p *Hash2Path) commit(path []byte, n node) node {
// if this path is clean, use available cached data
hash, dirty := n.cache()
if hash != nil && !dirty {
return hash
}
// Commit children, then parent, and remove the dirty flag.
switch cn := n.(type) {
case *shortNode:
// Commit child
collapsed := cn.copy()

// If the child is fullNode, recursively commit,
// otherwise it can only be hashNode or valueNode.
if _, ok := cn.Val.(*fullNode); ok {
collapsed.Val = h2p.commit(append(path, cn.Key...), cn.Val)
}
// The key needs to be copied, since we're adding it to the
// modified nodeset.
collapsed.Key = hexToCompact(cn.Key)
return collapsed
case *fullNode:
hashedKids := h2p.commitChildren(path, cn)
collapsed := cn.copy()
collapsed.Children = hashedKids

return collapsed
case hashNode:
return cn
default:
// nil, valuenode shouldn't be committed
panic(fmt.Sprintf("%T: invalid node: %v", n, n))
}
}
Loading

0 comments on commit f60e666

Please sign in to comment.