lmbdx integration as key-value pair database

.github/workflows/juno-test.yml

@@ -4,6 +4,7 @@ on:
     branches:
       - main
   pull_request:
+  workflow_dispatch:
 
 jobs:
   test:
@@ -18,6 +19,8 @@ jobs:
         uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go }}
+      - name: install deps
+        run: make install-deps
       - name: Tests
         run: make codecov-test
       - name: Benchmark
@@ -26,7 +29,7 @@ jobs:
         uses: codecov/codecov-action@v2
         with:
           fail_ci_if_error: true
-          files: ./coverage/coverage.txt
+          files: ./coverage/coverage.out
           flags: unittests
           name: codecov-umbrella
           verbose: true

.gitignore

@@ -1,2 +1,3 @@
 build/juno
 coverage/*
+courtney/

Makefile

@@ -20,9 +20,14 @@ test-cover: ## tests with coverage
 	go test -coverprofile=coverage/coverage.out -covermode=count ./...
 	go tool cover -html=coverage/coverage.out -o coverage/coverage.html
 
+install-deps: ## Install some project dependencies
+	git clone https://github.com/DemerzelSolutions/courtney
+	(cd courtney && go get  ./... && go build courtney.go)
+	go get ./...
+
 codecov-test:
 	mkdir -p coverage
-	go test -coverprofile=coverage/coverage.txt -covermode=atomic ./...
+	courtney/courtney -v -o coverage/coverage.out ./...
 
 gomod_tidy: ## add missing and remove unused modules
 	 go mod tidy

cmd/juno/main.go

@@ -4,6 +4,8 @@ import (
 	cmd "github.com/NethermindEth/juno/cmd/cli"
 )
 
+//var logger = log.GetLogger()
+
 func main() {
 	cmd.Execute()
 	//end := make(chan error)
@@ -23,5 +25,31 @@ func main() {
 	//	logger.With("With Error", err).Error("Failed to retrieve block")
 	//}
 	//logger.With("blockHash", block.BlockHash).Debug("Block Hash retrieved from provider, ")
+	//database := db.NewKeyValueDatabase("example", 0)
+	//
+	//val, err := database.Get([]byte("blockHash"))
+	//if err != nil {
+	//	logger.With("Error", err).Error("Error getting values from")
+	//	return
+	//}
+	//logger.With("blockHash", string(val)).Info("Got latest BlockHash Value from DB")
+	//
+	//if block.BlockHash == string(val) {
+	//	logger.Info("Still the same blockHash")
+	//	return
+	//}
+	//logger.Info("Storing the new blockHash")
+	//
+	//err = database.Put([]byte("blockHash"), []byte(block.BlockHash))
+	//if err != nil {
+	//	logger.With("Error", err).Error("Error putting in database")
+	//	return
+	//}
+	//n, err := database.NumberOfItems()
+	//if err != nil {
+	//	logger.With("Error", err).Error("Error getting the number of items in database")
+	//	return
+	//}
+	//logger.With("Number of Items on DB", n).Info("Got the number of items on DB")
 
 }

go.mod

@@ -3,30 +3,38 @@ module github.com/NethermindEth/juno
 go 1.17
 
 require (
-	github.com/ethereum/go-ethereum v1.10.14
-	github.com/spf13/cobra v0.0.3
+	github.com/ethereum/go-ethereum v1.10.16
+	github.com/sirupsen/logrus v1.8.1
+	github.com/spf13/cobra v1.3.0
 	github.com/spf13/viper v1.10.1
+	github.com/torquem-ch/mdbx-go v0.22.15
 	go.uber.org/zap v1.21.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
 	github.com/btcsuite/btcd v0.20.1-beta // indirect
+	github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14 // indirect
+	github.com/dave/brenda v1.1.0 // indirect
+	github.com/dave/courtney v0.3.0 // indirect
+	github.com/dave/patsy v0.0.0-20210517141501-957256f50cba // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
-	go.uber.org/atomic v1.9.0 // indirect
-	go.uber.org/multierr v1.7.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
-	golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect
+	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
 	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/tools v0.1.9 // indirect
 	gopkg.in/ini.v1 v1.66.2 // indirect
 )

internal/log/logging.go

@@ -7,18 +7,9 @@ import (
 
 var once sync.Once
 
-var logger *zap.Logger
 var sugaredLogger *zap.SugaredLogger
 
-// Sync flush any buffered log entry
-func Sync() {
-	err := logger.Sync()
-	if err != nil {
-		return
-	}
-}
-
-// GetLogger returns logger for the app
+// GetLogger returns logger for the app using Singleton Pattern Design
 func GetLogger() *zap.SugaredLogger {
 	once.Do(func() {
 		// See https://pkg.go.dev/go.uber.org/zap#hdr-Choosing_a_Logger for more configurations

pkg/db/buckets.go

@@ -0,0 +1 @@
+package db

pkg/db/database.go

@@ -0,0 +1,134 @@
+package db
+
+import (
+	"github.com/NethermindEth/juno/internal/log"
+	"github.com/torquem-ch/mdbx-go/mdbx"
+)
+
+var logger = log.GetLogger()
+
+// KeyValueDatabase Represent a middleware for mdbx key-value pair database
+type KeyValueDatabase struct {
+	env  *mdbx.Env
+	path string
+}
+
+// NewKeyValueDatabase Creates a new KeyValueDatabase
+func NewKeyValueDatabase(path string, flags uint) *KeyValueDatabase {
+	logger.With(
+		"Path", path,
+		"Flags", flags,
+	).Info("Creating new database")
+	env, err1 := mdbx.NewEnv()
+	if err1 != nil {
+		// notest
+		logger.With("Error", err1).Fatalf("Cannot create environment")
+		return nil
+	}
+	// Set Flags
+	// Based on https://github.com/torquem-ch/mdbx-go/blob/96f31f483af593377e52358a079e834256d5af55/mdbx/env_test.go#L495
+	err := env.SetOption(mdbx.OptMaxDB, 1024)
+	if err != nil {
+		// notest
+		logger.With("Error", err).Fatalf("Cannot set Options")
+		return nil
+	}
+	const pageSize = 4096
+	err = env.SetGeometry(-1, -1, 64*1024*pageSize, -1, -1, pageSize)
+	if err != nil {
+		// notest
+		logger.With("Error", err1).Fatalf("Cannot set Geometry")
+		return nil
+	}
+	err = env.Open(path, flags, 0664)
+	if err != nil {
+		// notest
+		logger.With("Error", err1).Fatalf("Cannot open env")
+		return nil
+	}
+	return &KeyValueDatabase{
+		env:  env,
+		path: path,
+	}
+}
+
+func (d KeyValueDatabase) Has(key []byte) (has bool, err error) {
+	val, err := d.getOne(key)
+	if err != nil {
+		return false, err
+	}
+	return val != nil, nil
+}
+
+func (d KeyValueDatabase) getOne(key []byte) (val []byte, err error) {
+	var db mdbx.DBI
+	if err := d.env.View(func(txn *mdbx.Txn) error {
+		db, err = txn.OpenRoot(mdbx.Create)
+
+		if err != nil {
+			return err
+		}
+		val, err = txn.Get(db, key)
+		if mdbx.IsNotFound(err) {
+			err = nil
+			return err
+		}
+		if err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return val, err
+}
+
+func (d KeyValueDatabase) Get(key []byte) ([]byte, error) {
+	logger.With("Key", key).Info("Getting value of provided key")
+	return d.getOne(key)
+}
+
+func (d KeyValueDatabase) Put(key, value []byte) error {
+	logger.With("Key", string(key)).Info("Putting value of provided key")
+	err := d.env.Update(func(txn *mdbx.Txn) (err error) {
+		logger.Debug("Open DBI")
+		dbi, err := txn.OpenRoot(mdbx.Create)
+
+		if err != nil {
+			return err
+		}
+		logger.Debug("Inserting on db")
+		return txn.Put(dbi, key, value, 0)
+	})
+	return err
+}
+
+func (d KeyValueDatabase) Delete(key []byte) error {
+	logger.With("Key", key).Info("Deleting value associated to provided key")
+	err := d.env.Update(func(txn *mdbx.Txn) (err error) {
+		db, err := txn.OpenRoot(mdbx.Create)
+		return txn.Del(db, key, nil)
+	})
+	return err
+}
+
+func (d KeyValueDatabase) NumberOfItems() (uint64, error) {
+	logger.Info("Getting the amount of items in the collection")
+	stats, err := d.env.Stat()
+	if err != nil {
+		return 0, err
+	}
+	return stats.Entries, err
+}
+
+func (d KeyValueDatabase) Begin() {
+	//TODO implement me
+}
+
+func (d KeyValueDatabase) Rollback() {
+	//TODO implement me
+}
+
+func (d KeyValueDatabase) Close() {
+	d.env.Close()
+}