-
Notifications
You must be signed in to change notification settings - Fork 171
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
lmbdx integration as key-value pair database (#80)
* Add interfaces for Database
* Add mdbx to go.{mod.sum}
* Modify interface
* Implementation of database interface
* Add tests files for database
* Add test for add key and solve small errors
* Solve small errors in database tests
* Add sirupsen/log for logging to app
* Add instructions for log in README
* Use logrus as logging in main
* Add debugging for encoding of chain ID
* Add logs for sn_keccak
* Add method for get the number of records inside DB interface
* [log] Add comments and logs and restructuration for implementation of new interface in DB
* Change name for kv-database initializer
* Add more logs to database and solve small typos
* Add first set of tests for database
* Remove unused lines
* Update tests for database to receive db path as params
* Add benchmarking for databases
* Set log level to error and debug in tests
* Add new functionalities to init
* Add comments for database_test
* Improve code-coverage for database tests
* Add zap to go modules as new logging library
* Create using Singleton pattern logger for app using zap
* Change in main new logging with zap for logrus
* Remove logrus from go mod
* Change zap for logrus in internal code
* Add docs for logging in readme using zap
* Update implementations to use With
* change logs from logrus to zap
* Modify comment in database log
* Add example about how to store in database
* Update image of the logs
* Add image to README
* Add image to README
* Update image of the logs
* Remove unused mutex in database
* Add courtney for code-coverage
* Solve code coverage issues
* Fix workflow to use courtney for code-coverage
* Change tests layout
* Update makefile courtney installation
* Add go get in makefile for project
* Add gopath to command
* Add gopath initialization in makefile
* Add initialization for gopath in makefile
* Change to install latest
* Add DemerzelSolutions fork as base for courtney coverage
* Install courtney using cli instead package
* Add go get to deps install
* Set .out instead .txt as coverage output
* Add workflow_dispatch trigger to juno-test workflow
Co-authored-by: AntiD2ta <mtpotro41@gmail.com>- Loading branch information
Showing
11 changed files
with
523 additions
and
28 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,3 @@ | ||
| build/juno | ||
| coverage/* | ||
| courtney/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| package db |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,134 @@ | ||
| package db | ||
|
|
||
| import ( | ||
| "github.com/NethermindEth/juno/internal/log" | ||
| "github.com/torquem-ch/mdbx-go/mdbx" | ||
| ) | ||
|
|
||
| var logger = log.GetLogger() | ||
|
|
||
| // KeyValueDatabase Represent a middleware for mdbx key-value pair database | ||
| type KeyValueDatabase struct { | ||
| env *mdbx.Env | ||
| path string | ||
| } | ||
|
|
||
| // NewKeyValueDatabase Creates a new KeyValueDatabase | ||
| func NewKeyValueDatabase(path string, flags uint) *KeyValueDatabase { | ||
| logger.With( | ||
| "Path", path, | ||
| "Flags", flags, | ||
| ).Info("Creating new database") | ||
| env, err1 := mdbx.NewEnv() | ||
| if err1 != nil { | ||
| // notest | ||
| logger.With("Error", err1).Fatalf("Cannot create environment") | ||
| return nil | ||
| } | ||
| // Set Flags | ||
| // Based on https://github.com/torquem-ch/mdbx-go/blob/96f31f483af593377e52358a079e834256d5af55/mdbx/env_test.go#L495 | ||
| err := env.SetOption(mdbx.OptMaxDB, 1024) | ||
| if err != nil { | ||
| // notest | ||
| logger.With("Error", err).Fatalf("Cannot set Options") | ||
| return nil | ||
| } | ||
| const pageSize = 4096 | ||
| err = env.SetGeometry(-1, -1, 64*1024*pageSize, -1, -1, pageSize) | ||
| if err != nil { | ||
| // notest | ||
| logger.With("Error", err1).Fatalf("Cannot set Geometry") | ||
| return nil | ||
| } | ||
| err = env.Open(path, flags, 0664) | ||
| if err != nil { | ||
| // notest | ||
| logger.With("Error", err1).Fatalf("Cannot open env") | ||
| return nil | ||
| } | ||
| return &KeyValueDatabase{ | ||
| env: env, | ||
| path: path, | ||
| } | ||
| } | ||
|
|
||
| func (d KeyValueDatabase) Has(key []byte) (has bool, err error) { | ||
| val, err := d.getOne(key) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| return val != nil, nil | ||
| } | ||
|
|
||
| func (d KeyValueDatabase) getOne(key []byte) (val []byte, err error) { | ||
| var db mdbx.DBI | ||
| if err := d.env.View(func(txn *mdbx.Txn) error { | ||
| db, err = txn.OpenRoot(mdbx.Create) | ||
|
|
||
| if err != nil { | ||
| return err | ||
| } | ||
| val, err = txn.Get(db, key) | ||
| if mdbx.IsNotFound(err) { | ||
| err = nil | ||
| return err | ||
| } | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return nil | ||
| }); err != nil { | ||
| return nil, err | ||
| } | ||
| return val, err | ||
| } | ||
|
|
||
| func (d KeyValueDatabase) Get(key []byte) ([]byte, error) { | ||
| logger.With("Key", key).Info("Getting value of provided key") | ||
| return d.getOne(key) | ||
| } | ||
|
|
||
| func (d KeyValueDatabase) Put(key, value []byte) error { | ||
| logger.With("Key", string(key)).Info("Putting value of provided key") | ||
| err := d.env.Update(func(txn *mdbx.Txn) (err error) { | ||
| logger.Debug("Open DBI") | ||
| dbi, err := txn.OpenRoot(mdbx.Create) | ||
|
|
||
| if err != nil { | ||
| return err | ||
| } | ||
| logger.Debug("Inserting on db") | ||
| return txn.Put(dbi, key, value, 0) | ||
| }) | ||
| return err | ||
| } | ||
|
|
||
| func (d KeyValueDatabase) Delete(key []byte) error { | ||
| logger.With("Key", key).Info("Deleting value associated to provided key") | ||
| err := d.env.Update(func(txn *mdbx.Txn) (err error) { | ||
| db, err := txn.OpenRoot(mdbx.Create) | ||
| return txn.Del(db, key, nil) | ||
| }) | ||
| return err | ||
| } | ||
|
|
||
| func (d KeyValueDatabase) NumberOfItems() (uint64, error) { | ||
| logger.Info("Getting the amount of items in the collection") | ||
| stats, err := d.env.Stat() | ||
| if err != nil { | ||
| return 0, err | ||
| } | ||
| return stats.Entries, err | ||
| } | ||
|
|
||
| func (d KeyValueDatabase) Begin() { | ||
| //TODO implement me | ||
| } | ||
|
|
||
| func (d KeyValueDatabase) Rollback() { | ||
| //TODO implement me | ||
| } | ||
|
|
||
| func (d KeyValueDatabase) Close() { | ||
| d.env.Close() | ||
| } |
Oops, something went wrong.