From ab8f71dc18445b2730a6f83ac1a9f2fa11e6bc2a Mon Sep 17 00:00:00 2001 From: Marcos Antonio Maceo <35319980+stdevMac@users.noreply.github.com> Date: Tue, 1 Mar 2022 22:46:27 -0500 Subject: [PATCH] lmbdx integration as key-value pair database (#80) * Add interfaces for Database * Add mdbx to go.{mod.sum} * Modify interface * Implementation of database interface * Add tests files for database * Add test for add key and solve small errors * Solve small errors in database tests * Add sirupsen/log for logging to app * Add instructions for log in README * Use logrus as logging in main * Add debugging for encoding of chain ID * Add logs for sn_keccak * Add method for get the number of records inside DB interface * [log] Add comments and logs and restructuration for implementation of new interface in DB * Change name for kv-database initializer * Add more logs to database and solve small typos * Add first set of tests for database * Remove unused lines * Update tests for database to receive db path as params * Add benchmarking for databases * Set log level to error and debug in tests * Add new functionalities to init * Add comments for database_test * Improve code-coverage for database tests * Add zap to go modules as new logging library * Create using Singleton pattern logger for app using zap * Change in main new logging with zap for logrus * Remove logrus from go mod * Change zap for logrus in internal code * Add docs for logging in readme using zap * Update implementations to use With * change logs from logrus to zap * Modify comment in database log * Add example about how to store in database * Update image of the logs * Add image to README * Add image to README * Update image of the logs * Remove unused mutex in database * Add courtney for code-coverage * Solve code coverage issues * Fix workflow to use courtney for code-coverage * Change tests layout * Update makefile courtney installation * Add go get in makefile for project * Add gopath to command * Add gopath initialization in makefile * Add initialization for gopath in makefile * Change to install latest * Add DemerzelSolutions fork as base for courtney coverage * Install courtney using cli instead package * Add go get to deps install * Set .out instead .txt as coverage output * Add workflow_dispatch trigger to juno-test workflow Co-authored-by: AntiD2ta --- .github/workflows/juno-test.yml | 5 +- .gitignore | 1 + Makefile | 7 +- cmd/juno/main.go | 28 ++++ go.mod | 18 ++- go.sum | 55 +++++-- internal/log/logging.go | 11 +- pkg/db/buckets.go | 1 + pkg/db/database.go | 134 ++++++++++++++++ pkg/db/database_test.go | 270 ++++++++++++++++++++++++++++++++ pkg/db/interface.go | 21 +++ 11 files changed, 523 insertions(+), 28 deletions(-) create mode 100644 pkg/db/buckets.go create mode 100644 pkg/db/database.go create mode 100644 pkg/db/database_test.go create mode 100644 pkg/db/interface.go diff --git a/.github/workflows/juno-test.yml b/.github/workflows/juno-test.yml index ed3574d694..fbee204427 100644 --- a/.github/workflows/juno-test.yml +++ b/.github/workflows/juno-test.yml @@ -4,6 +4,7 @@ on: branches: - main pull_request: + workflow_dispatch: jobs: test: @@ -18,6 +19,8 @@ jobs: uses: actions/setup-go@v2 with: go-version: ${{ matrix.go }} + - name: install deps + run: make install-deps - name: Tests run: make codecov-test - name: Benchmark @@ -26,7 +29,7 @@ jobs: uses: codecov/codecov-action@v2 with: fail_ci_if_error: true - files: ./coverage/coverage.txt + files: ./coverage/coverage.out flags: unittests name: codecov-umbrella verbose: true diff --git a/.gitignore b/.gitignore index 53937377b8..8ec99244a6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ build/juno coverage/* +courtney/ diff --git a/Makefile b/Makefile index ec79367662..489f772b46 100644 --- a/Makefile +++ b/Makefile @@ -20,9 +20,14 @@ test-cover: ## tests with coverage go test -coverprofile=coverage/coverage.out -covermode=count ./... go tool cover -html=coverage/coverage.out -o coverage/coverage.html +install-deps: ## Install some project dependencies + git clone https://github.com/DemerzelSolutions/courtney + (cd courtney && go get ./... && go build courtney.go) + go get ./... + codecov-test: mkdir -p coverage - go test -coverprofile=coverage/coverage.txt -covermode=atomic ./... + courtney/courtney -v -o coverage/coverage.out ./... gomod_tidy: ## add missing and remove unused modules go mod tidy diff --git a/cmd/juno/main.go b/cmd/juno/main.go index c237268104..1fbee570d1 100644 --- a/cmd/juno/main.go +++ b/cmd/juno/main.go @@ -4,6 +4,8 @@ import ( cmd "github.com/NethermindEth/juno/cmd/cli" ) +//var logger = log.GetLogger() + func main() { cmd.Execute() //end := make(chan error) @@ -23,5 +25,31 @@ func main() { // logger.With("With Error", err).Error("Failed to retrieve block") //} //logger.With("blockHash", block.BlockHash).Debug("Block Hash retrieved from provider, ") + //database := db.NewKeyValueDatabase("example", 0) + // + //val, err := database.Get([]byte("blockHash")) + //if err != nil { + // logger.With("Error", err).Error("Error getting values from") + // return + //} + //logger.With("blockHash", string(val)).Info("Got latest BlockHash Value from DB") + // + //if block.BlockHash == string(val) { + // logger.Info("Still the same blockHash") + // return + //} + //logger.Info("Storing the new blockHash") + // + //err = database.Put([]byte("blockHash"), []byte(block.BlockHash)) + //if err != nil { + // logger.With("Error", err).Error("Error putting in database") + // return + //} + //n, err := database.NumberOfItems() + //if err != nil { + // logger.With("Error", err).Error("Error getting the number of items in database") + // return + //} + //logger.With("Number of Items on DB", n).Info("Got the number of items on DB") } diff --git a/go.mod b/go.mod index bb14c95bb3..616d400319 100644 --- a/go.mod +++ b/go.mod @@ -3,30 +3,38 @@ module github.com/NethermindEth/juno go 1.17 require ( - github.com/ethereum/go-ethereum v1.10.14 - github.com/spf13/cobra v0.0.3 + github.com/ethereum/go-ethereum v1.10.16 + github.com/sirupsen/logrus v1.8.1 + github.com/spf13/cobra v1.3.0 github.com/spf13/viper v1.10.1 + github.com/torquem-ch/mdbx-go v0.22.15 go.uber.org/zap v1.21.0 gopkg.in/yaml.v2 v2.4.0 ) require ( github.com/btcsuite/btcd v0.20.1-beta // indirect + github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14 // indirect + github.com/dave/brenda v1.1.0 // indirect + github.com/dave/courtney v0.3.0 // indirect + github.com/dave/patsy v0.0.0-20210517141501-957256f50cba // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.7.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect - golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect + golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect golang.org/x/text v0.3.7 // indirect + golang.org/x/tools v0.1.9 // indirect gopkg.in/ini.v1 v1.66.2 // indirect ) diff --git a/go.sum b/go.sum index 8666cf0401..4c1173179b 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,7 @@ cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aD cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= @@ -139,14 +140,23 @@ github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= +github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14 h1:YI1gOOdmMk3xodBao7fehcvoZsEeOyy/cfhlpCSPgM4= +github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14/go.mod h1:Sth2QfxfATb/nW4EsrSi2KyJmbcniZ8TgTaji17D6ms= +github.com/dave/brenda v1.1.0 h1:Sl1LlwXnbw7xMhq3y2x11McFu43AjDcwkllxxgZ3EZw= +github.com/dave/brenda v1.1.0/go.mod h1:4wCUr6gSlu5/1Tk7akE5X7UorwiQ8Rij0SKH3/BGMOM= +github.com/dave/courtney v0.3.0 h1:8aR1os2ImdIQf3Zj4oro+lD/L4Srb5VwGefqZ/jzz7U= +github.com/dave/courtney v0.3.0/go.mod h1:BAv3hA06AYfNUjfjQr+5gc6vxeBVOupLqrColj+QSD8= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/dave/patsy v0.0.0-20210517141501-957256f50cba h1:1o36L4EKbZzazMk8iGC4kXpVnZ6TPxR2mZ9qVKjNNAs= +github.com/dave/patsy v0.0.0-20210517141501-957256f50cba/go.mod h1:qfR88CgEGLoiqDaE+xxDCi5QA5v4vUoW0UCX2Nd5Tlc= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -169,8 +179,8 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= -github.com/ethereum/go-ethereum v1.10.14 h1:EJ/ucQzFlgKgwblIwU8R6ABnZ9kgUnIG2+Q1tiSrt4M= -github.com/ethereum/go-ethereum v1.10.14/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= +github.com/ethereum/go-ethereum v1.10.16 h1:3oPrumn0bCW/idjcxMn5YYVCdK7VzJYIvwGZUGLEaoc= +github.com/ethereum/go-ethereum v1.10.16/go.mod h1:Anj6cxczl+AHy63o4X9O8yWNHuN5wMpfb8MAnHkWn7Y= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= @@ -287,8 +297,9 @@ github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -316,8 +327,11 @@ github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:i github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= @@ -340,7 +354,7 @@ github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19y github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= -github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -357,7 +371,7 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= -github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -411,6 +425,7 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= @@ -484,7 +499,9 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDNxqmyJ6RfDFM= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= @@ -494,6 +511,8 @@ github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -503,13 +522,15 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= +github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= @@ -529,6 +550,8 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= +github.com/torquem-ch/mdbx-go v0.22.15 h1:mFMP1lC0716ifEAvg8FoJ1iJCXww0UFe4aRgbfTRAYY= +github.com/torquem-ch/mdbx-go v0.22.15/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= @@ -554,21 +577,20 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= -go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -625,6 +647,7 @@ golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -699,6 +722,7 @@ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -772,8 +796,11 @@ golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -850,6 +877,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.9 h1:j9KsMiaP1c3B0OTQGth0/k+miLGTgLsAFUCrF2vLcF8= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -891,6 +920,7 @@ google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqiv google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -960,6 +990,8 @@ google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -988,6 +1020,7 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/internal/log/logging.go b/internal/log/logging.go index 1351434055..a3b71bff8e 100644 --- a/internal/log/logging.go +++ b/internal/log/logging.go @@ -7,18 +7,9 @@ import ( var once sync.Once -var logger *zap.Logger var sugaredLogger *zap.SugaredLogger -// Sync flush any buffered log entry -func Sync() { - err := logger.Sync() - if err != nil { - return - } -} - -// GetLogger returns logger for the app +// GetLogger returns logger for the app using Singleton Pattern Design func GetLogger() *zap.SugaredLogger { once.Do(func() { // See https://pkg.go.dev/go.uber.org/zap#hdr-Choosing_a_Logger for more configurations diff --git a/pkg/db/buckets.go b/pkg/db/buckets.go new file mode 100644 index 0000000000..3a49c63e35 --- /dev/null +++ b/pkg/db/buckets.go @@ -0,0 +1 @@ +package db diff --git a/pkg/db/database.go b/pkg/db/database.go new file mode 100644 index 0000000000..cb56262798 --- /dev/null +++ b/pkg/db/database.go @@ -0,0 +1,134 @@ +package db + +import ( + "github.com/NethermindEth/juno/internal/log" + "github.com/torquem-ch/mdbx-go/mdbx" +) + +var logger = log.GetLogger() + +// KeyValueDatabase Represent a middleware for mdbx key-value pair database +type KeyValueDatabase struct { + env *mdbx.Env + path string +} + +// NewKeyValueDatabase Creates a new KeyValueDatabase +func NewKeyValueDatabase(path string, flags uint) *KeyValueDatabase { + logger.With( + "Path", path, + "Flags", flags, + ).Info("Creating new database") + env, err1 := mdbx.NewEnv() + if err1 != nil { + // notest + logger.With("Error", err1).Fatalf("Cannot create environment") + return nil + } + // Set Flags + // Based on https://github.com/torquem-ch/mdbx-go/blob/96f31f483af593377e52358a079e834256d5af55/mdbx/env_test.go#L495 + err := env.SetOption(mdbx.OptMaxDB, 1024) + if err != nil { + // notest + logger.With("Error", err).Fatalf("Cannot set Options") + return nil + } + const pageSize = 4096 + err = env.SetGeometry(-1, -1, 64*1024*pageSize, -1, -1, pageSize) + if err != nil { + // notest + logger.With("Error", err1).Fatalf("Cannot set Geometry") + return nil + } + err = env.Open(path, flags, 0664) + if err != nil { + // notest + logger.With("Error", err1).Fatalf("Cannot open env") + return nil + } + return &KeyValueDatabase{ + env: env, + path: path, + } +} + +func (d KeyValueDatabase) Has(key []byte) (has bool, err error) { + val, err := d.getOne(key) + if err != nil { + return false, err + } + return val != nil, nil +} + +func (d KeyValueDatabase) getOne(key []byte) (val []byte, err error) { + var db mdbx.DBI + if err := d.env.View(func(txn *mdbx.Txn) error { + db, err = txn.OpenRoot(mdbx.Create) + + if err != nil { + return err + } + val, err = txn.Get(db, key) + if mdbx.IsNotFound(err) { + err = nil + return err + } + if err != nil { + return err + } + return nil + }); err != nil { + return nil, err + } + return val, err +} + +func (d KeyValueDatabase) Get(key []byte) ([]byte, error) { + logger.With("Key", key).Info("Getting value of provided key") + return d.getOne(key) +} + +func (d KeyValueDatabase) Put(key, value []byte) error { + logger.With("Key", string(key)).Info("Putting value of provided key") + err := d.env.Update(func(txn *mdbx.Txn) (err error) { + logger.Debug("Open DBI") + dbi, err := txn.OpenRoot(mdbx.Create) + + if err != nil { + return err + } + logger.Debug("Inserting on db") + return txn.Put(dbi, key, value, 0) + }) + return err +} + +func (d KeyValueDatabase) Delete(key []byte) error { + logger.With("Key", key).Info("Deleting value associated to provided key") + err := d.env.Update(func(txn *mdbx.Txn) (err error) { + db, err := txn.OpenRoot(mdbx.Create) + return txn.Del(db, key, nil) + }) + return err +} + +func (d KeyValueDatabase) NumberOfItems() (uint64, error) { + logger.Info("Getting the amount of items in the collection") + stats, err := d.env.Stat() + if err != nil { + return 0, err + } + return stats.Entries, err +} + +func (d KeyValueDatabase) Begin() { + //TODO implement me +} + +func (d KeyValueDatabase) Rollback() { + //TODO implement me +} + +func (d KeyValueDatabase) Close() { + d.env.Close() +} diff --git a/pkg/db/database_test.go b/pkg/db/database_test.go new file mode 100644 index 0000000000..f9eeeed42b --- /dev/null +++ b/pkg/db/database_test.go @@ -0,0 +1,270 @@ +package db + +import ( + log "github.com/sirupsen/logrus" + "strconv" + "testing" +) + +var ( + KeyValueTest = map[string]string{} +) + +func init() { + log.SetLevel(log.ErrorLevel) + for i := 0; i < 350; i++ { + val := strconv.Itoa(i) + KeyValueTest["key"+val] = "value" + val + } +} + +// setupDatabaseForTest creates a new Database for Tests +func setupDatabaseForTest(path string) *KeyValueDatabase { + return NewKeyValueDatabase(path, 0) +} + +// TestAddKey Check that a single value is inserted without error +func TestAddKey(t *testing.T) { + db := setupDatabaseForTest(t.TempDir()) + err := db.Put([]byte("key"), []byte("value")) + if err != nil { + t.Log(err) + t.Fail() + } +} + +// TestNumberOfItems Checks that in every moment the collection contains the right amount of items +func TestNumberOfItems(t *testing.T) { + db := setupDatabaseForTest(t.TempDir()) + n, err := db.NumberOfItems() + if err != nil { + t.Log(err) + t.Fail() + return + } + if n != 0 { + t.Log(err) + t.Fail() + return + } + for k, v := range KeyValueTest { + err := db.Put([]byte(k), []byte(v)) + if err != nil { + t.Log(err) + t.Fail() + return + } + } + n, err = db.NumberOfItems() + if err != nil { + t.Log(err) + t.Fail() + return + } + if int(n) != len(KeyValueTest) { + t.Log(err) + t.Fail() + } +} + +// TestAddMultipleKeys Checks that after insert some keys the collection contains the right amount of items +func TestAddMultipleKeys(t *testing.T) { + db := setupDatabaseForTest(t.TempDir()) + for k, v := range KeyValueTest { + err := db.Put([]byte(k), []byte(v)) + if err != nil { + t.Log(err) + t.Fail() + return + } + } + n, err := db.NumberOfItems() + if err != nil { + t.Log(err) + t.Fail() + return + } + if int(n) != len(KeyValueTest) { + t.Log(err) + t.Fail() + } +} + +// TestHasKey Check that one key exist after insertion +func TestHasKey(t *testing.T) { + db := setupDatabaseForTest(t.TempDir()) + goodKey := []byte("good_key") + err := db.Put(goodKey, []byte("value")) + if err != nil { + t.Log(err) + t.Fail() + return + } + has, err := db.Has(goodKey) + if err != nil { + t.Log(err) + t.Fail() + return + } + if !has { + t.Log(err) + t.Fail() + return + } +} + +// TestHasNotKey Check that a key don't exist +func TestHasNotKey(t *testing.T) { + db := setupDatabaseForTest(t.TempDir()) + goodKey := []byte("good_key") + badKey := []byte("bad_key") + err := db.Put(goodKey, []byte("value")) + if err != nil { + t.Log(err) + t.Fail() + return + } + has2, err := db.Has(badKey) + if err != nil { + t.Log(err) + t.Fail() + return + } + if has2 { + t.Log(err) + t.Fail() + } +} + +// TestGetKey Check that a key is property retrieved +func TestGetKey(t *testing.T) { + db := setupDatabaseForTest(t.TempDir()) + goodKey := []byte("good_key") + goodValue := []byte("value") + err := db.Put(goodKey, goodValue) + if err != nil { + t.Log(err) + t.Fail() + return + } + val, err := db.Get(goodKey) + if err != nil { + t.Log(err) + t.Fail() + return + } + if string(val) != string(goodValue) { + t.Log(err) + t.Fail() + } +} + +// TestGetNotKey Check that a key don't exist and what happen if it doesn't exist +func TestGetNotKey(t *testing.T) { + db := setupDatabaseForTest(t.TempDir()) + goodKey := []byte("good_key") + goodValue := []byte("value") + badKey := []byte("bad_key") + err := db.Put(goodKey, goodValue) + if err != nil { + t.Log(err) + t.Fail() + } + key, err := db.Get(badKey) + if err != nil { + t.Log(err) + t.Fail() + return + } + if key != nil { + t.Log(err) + t.Fail() + return + } +} + +func TestDelete(t *testing.T) { + db := setupDatabaseForTest(t.TempDir()) + goodKey := []byte("good_key") + goodValue := []byte("value") + err := db.Put(goodKey, goodValue) + if err != nil { + t.Log(err) + t.Fail() + } + err = db.Delete(goodKey) + if err != nil { + t.Log(err) + t.Fail() + return + } + key, err := db.Has(goodKey) + if key { + t.Log(err) + t.Fail() + return + } +} + +func TestBegin(t *testing.T) { + db := setupDatabaseForTest(t.TempDir()) + db.Begin() +} +func TestRollBack(t *testing.T) { + db := setupDatabaseForTest(t.TempDir()) + db.Rollback() +} +func TestClose(t *testing.T) { + db := setupDatabaseForTest(t.TempDir()) + db.Close() +} + +// BenchmarkEntriesInDatabase Benchmark the entry of key-value pairs to the db +func BenchmarkEntriesInDatabase(b *testing.B) { + log.SetLevel(log.ErrorLevel) + db := setupDatabaseForTest(b.TempDir()) + for i := 0; i < b.N; i++ { + val := []byte(strconv.Itoa(i)) + err := db.Put(val, val) + if err != nil { + b.Error("Benchmarking fails, error storing values") + return + } + } + n, err := db.NumberOfItems() + if err != nil { + b.Errorf("Benchmarking fails, error getting the number of items: %s\n", err) + b.Fail() + return + } + if int(n) != b.N { + b.Error("Benchmarking fails, mismatch between number of items to insert and the number inside db") + b.Fail() + return + } + +} + +// BenchmarkConsultsToDatabase Benchmark the consult to a db +func BenchmarkConsultsToDatabase(b *testing.B) { + db := setupDatabaseForTest(b.TempDir()) + for i := 0; i < b.N; i++ { + val := []byte(strconv.Itoa(i)) + err := db.Put(val, val) + if err != nil { + b.Error("Benchmarking fails, error storing values") + b.Fail() + return + } + get, err := db.Get(val) + if err != nil { + b.Errorf("Benchmarking fails, error getting values: %s\n", err) + b.Fail() + return + } + if string(get) != string(val) { + b.Error("Benchmarking fails, mismatch between expected return value and returned value") + b.Fail() + } + } +} diff --git a/pkg/db/interface.go b/pkg/db/interface.go new file mode 100644 index 0000000000..e4311184f2 --- /dev/null +++ b/pkg/db/interface.go @@ -0,0 +1,21 @@ +package db + +type Databaser interface { + // Has Check that the key provided exists in the collection + Has(key []byte) (bool, error) + + // Get Returns the value associated to the provided key or returns an error otherwise + Get(key []byte) ([]byte, error) + + // Put Insert the key-value pair into the collection + Put(key, value []byte) error + + // Delete Remove a previous inserted key, otherwise nothing happen + Delete(key []byte) error + + // NumberOfItems return the number of items in the collection + NumberOfItems() (uint64, error) + Begin() + Rollback() + Close() +}