From 5c2309b60ee6d383bbd71425b4397e22c2991277 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 26 Mar 2020 20:34:18 -0700 Subject: [PATCH 01/12] initial fuzzing harness --- fuzz/cmd/run/main.go | 50 +++++++++ fuzz/fuzzer.go | 251 +++++++++++++++++++++++++++++++++++++++++++ go.mod | 5 +- go.sum | 75 +++++++++++++ 4 files changed, 379 insertions(+), 2 deletions(-) create mode 100644 fuzz/cmd/run/main.go create mode 100644 fuzz/fuzzer.go diff --git a/fuzz/cmd/run/main.go b/fuzz/cmd/run/main.go new file mode 100644 index 0000000..2c4dceb --- /dev/null +++ b/fuzz/cmd/run/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "bufio" + "fmt" + "io/ioutil" + "os" + + ds "github.com/ipfs/go-datastore" + badger "github.com/ipfs/go-ds-badger" + fuzzer "github.com/ipfs/go-datastore/fuzz" + + "github.com/spf13/pflag" +) + +var input *string = pflag.StringP("input", "i", "", "file to read input from (stdin used if not specified)") +var db *string = pflag.StringP("database", "d", "badger", "database to fuzz") +var dbFile *string = pflag.StringP("file", "f", "tmp", "where the db instace should live on disk") +var threads *int = pflag.IntP("threads", "t", 1, "concurrent threads") + +func main() { + pflag.Parse() + + fuzzer.Threads = *threads + if *db == "badger" { + fuzzer.DsOpener = func() ds.TxnDatastore { + d, err := badger.NewDatastore(*dbFile, &badger.DefaultOptions) + if err != nil { + panic("could not create db instance") + } + return d + } + } else { + // TODO + panic("unknown database") + } + + if *input != "" { + dat, err := ioutil.ReadFile(*input) + if err != nil { + fmt.Fprintf(os.Stderr, "could not read %s: %v\n", *input, err) + return + } + ret := fuzzer.Fuzz(dat) + os.Exit(ret) + } else { + ret := fuzzer.FuzzStream(bufio.NewReader(os.Stdin)) + os.Exit(ret) + } +} diff --git a/fuzz/fuzzer.go b/fuzz/fuzzer.go new file mode 100644 index 0000000..8a808eb --- /dev/null +++ b/fuzz/fuzzer.go @@ -0,0 +1,251 @@ +package fuzzer + +import ( + "context" + "io" + "sync/atomic" + + ds "github.com/ipfs/go-datastore" + "github.com/ipfs/go-ds-badger" +) + +// DsOpener is the concrete datastore. that Fuzz will fuzz against. +var DsOpener func() ds.TxnDatastore +var dsInst ds.TxnDatastore + +// Threads is a measure of concurrency. +var Threads int + +func init() { + DsOpener = func() ds.TxnDatastore { + d, _ := badger.NewDatastore("tmp", &badger.DefaultOptions) + return d + } + Threads = 1 + + keyCache[0] = ds.NewKey("/") + cachedKeys = 1 +} + +func setup() ([]chan byte, context.CancelFunc) { + // TODO: dynamic thread starting. + ctx, cncl := context.WithCancel(context.Background()) + + if dsInst != nil { + dsInst.Close() + } + dsInst = DsOpener() + + drivers := make([]chan byte, Threads) + for i := 0; i < Threads; i++ { + drivers[i] = make(chan byte, 15) + go threadDriver(ctx, drivers[i]) + } + return drivers, cncl +} + +// Fuzz is a go-fuzzer compatible input point for replaying +// data (interpreted as a script of commands) +// to a chosen ipfs datastore implementation +func Fuzz(data []byte) int { + drivers, cncl := setup() + drive(drivers, data) + for i := 0; i < Threads; i++ { + close(drivers[i]) + } + cncl() + dsInst.Close() + return 0 +} + +func drive(drivers []chan byte, data []byte) { + for i, b := range data { + drivers[i%Threads] <- b + } +} + +// FuzzStream does the same as fuzz but with streaming input +func FuzzStream(data io.Reader) int { + drivers, cncl := setup() + b := make([]byte, 4096) + for { + n, _ := data.Read(b) + if n == 0 { + break + } + drive(drivers, b[:n]) + } + for i := 0; i < Threads; i++ { + close(drivers[i]) + } + cncl() + dsInst.Close() + return 0 +} + +type op byte + +const ( + opNone op = iota + opGet + opHas + opGetSize + opQuery + opPut + opDelete + opNewTX + opCommitTX + opDiscardTX + opMax +) + +type state struct { + op + keyReady bool + key ds.Key + valReady bool + val []byte + reader ds.Read + writer ds.Write + txn ds.Txn +} + +func threadDriver(ctx context.Context, cmnds chan byte) error { + s := state{} + s.reader = dsInst + s.writer = dsInst + + for { + select { + case c, ok := <-cmnds: + if !ok { + return nil + } + _ = nextState(&s, c) + case <-ctx.Done(): + return nil + } + } +} + +func nextState(s *state, c byte) error { + if s.op == opNone { + s.op = op(c) % opMax + return nil + } else if s.op == opGet { + if !s.keyReady { + return makeKey(s, c) + } + s.reader.Get(s.key) + reset(s) + return nil + } else if s.op == opHas { + if !s.keyReady { + return makeKey(s, c) + } + s.reader.Has(s.key) + reset(s) + return nil + } else if s.op == opGetSize { + if !s.keyReady { + return makeKey(s, c) + } + s.reader.GetSize(s.key) + reset(s) + return nil + } else if s.op == opQuery { + // TODO + } else if s.op == opPut { + if !s.keyReady { + return makeKey(s, c) + } + if !s.valReady { + return makeValue(s, c) + } + s.writer.Put(s.key, s.val) + reset(s) + return nil + } else if s.op == opDelete { + if !s.keyReady { + return makeKey(s, c) + } + s.writer.Delete(s.key) + reset(s) + return nil + } else if s.op == opNewTX { + if s.txn == nil { + s.txn, _ = dsInst.NewTransaction(((c & 1) == 1)) + if (c & 1) != 1 { // read+write + s.writer = s.txn + } + s.reader = s.txn + } + reset(s) + return nil + } else if s.op == opCommitTX { + if s.txn != nil { + s.txn.Discard() + s.txn = nil + s.reader = dsInst + s.writer = dsInst + } + reset(s) + return nil + } else if s.op == opDiscardTX { + if s.txn != nil { + s.txn.Discard() + s.txn = nil + s.reader = dsInst + s.writer = dsInst + } + reset(s) + return nil + } + return nil +} + +func reset(s *state) { + s.op = opNone + s.keyReady = false + s.key = ds.RawKey("") + s.valReady = false +} + +var keyCache [128]ds.Key +var cachedKeys int32 + +func makeKey(s *state, c byte) error { + keys := atomic.LoadInt32(&cachedKeys) + if keys > 128 { + keys = 128 + } + if c&1 == 1 { + // 50% chance we want to-reuse an existing key + s.key = keyCache[(c>>1)%byte(keys)] + s.keyReady = true + } else { + s.key = ds.RandomKey() + // half the time we'll make it a child of an existing key + if c&2 == 2 { + s.key = keyCache[(c>>1)%byte(keys)].Child(s.key) + } + // new key + if keys < 128 { + keys = atomic.AddInt32(&cachedKeys, 1) + if keys >= 128 { + atomic.StoreInt32(&cachedKeys, 128) + } else { + keyCache[keys-1] = s.key + } + } + s.keyReady = true + } + return nil +} + +func makeValue(s *state, c byte) error { + s.val = make([]byte, c) + s.val[0] = 1 + s.valReady = true + return nil +} diff --git a/go.mod b/go.mod index 709a9c1..4be9a76 100644 --- a/go.mod +++ b/go.mod @@ -2,9 +2,10 @@ module github.com/ipfs/go-datastore require ( github.com/google/uuid v1.1.1 + github.com/ipfs/go-ds-badger v0.2.1 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8 - github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8 - github.com/kr/pretty v0.1.0 // indirect + github.com/jbenet/goprocess v0.1.3 + github.com/spf13/pflag v1.0.3 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 ) diff --git a/go.sum b/go.sum index d128972..7add6d6 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,91 @@ +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-ds-badger v0.2.1 h1:RsC9DDlwFhFdfT+s2PeC8joxbSp2YMufK8w/RBOxKtk= +github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8 h1:NAviDvJ0WXgD+yiL2Rj35AmnfgI11+pHXbdciD917U0= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= +github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= +github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8 h1:bspPhN+oKYFk5fcGNuQzp6IGzYQSenLEgH3s6jkXrWw= github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= +github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= +github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= +github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 5da85dd895fded16e46289cbf10aabbb83f1bef4 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 27 Mar 2020 10:02:23 -0700 Subject: [PATCH 02/12] different db's per instance --- fuzz/README.md | 6 ++++++ fuzz/fuzzer.go | 39 ++++++++++++++++++++++++++++----------- go.mod | 1 + go.sum | 2 ++ 4 files changed, 37 insertions(+), 11 deletions(-) create mode 100644 fuzz/README.md diff --git a/fuzz/README.md b/fuzz/README.md new file mode 100644 index 0000000..686e549 --- /dev/null +++ b/fuzz/README.md @@ -0,0 +1,6 @@ +IPFS Datastore Fuzzer +==== + +The fuzzer provides a [go fuzzer](https://github.com/dvyukov/go-fuzz) interface +to the Datastore interface. This can be used for fuzz testing of datastore +implementations. diff --git a/fuzz/fuzzer.go b/fuzz/fuzzer.go index 8a808eb..4cd5f04 100644 --- a/fuzz/fuzzer.go +++ b/fuzz/fuzzer.go @@ -3,23 +3,31 @@ package fuzzer import ( "context" "io" + "io/ioutil" + "os" + "sync" "sync/atomic" ds "github.com/ipfs/go-datastore" - "github.com/ipfs/go-ds-badger" + badger "github.com/ipfs/go-ds-badger" ) +type donefunc func() error + // DsOpener is the concrete datastore. that Fuzz will fuzz against. -var DsOpener func() ds.TxnDatastore +var DsOpener func() (ds.TxnDatastore, donefunc) var dsInst ds.TxnDatastore +var df donefunc // Threads is a measure of concurrency. var Threads int +var wg sync.WaitGroup func init() { - DsOpener = func() ds.TxnDatastore { - d, _ := badger.NewDatastore("tmp", &badger.DefaultOptions) - return d + DsOpener = func() (ds.TxnDatastore, donefunc) { + dir, _ := ioutil.TempDir("", "fuzz*") + d, _ := badger.NewDatastore(dir, &badger.DefaultOptions) + return d, func() error { return os.RemoveAll(dir) } } Threads = 1 @@ -31,11 +39,10 @@ func setup() ([]chan byte, context.CancelFunc) { // TODO: dynamic thread starting. ctx, cncl := context.WithCancel(context.Background()) - if dsInst != nil { - dsInst.Close() - } - dsInst = DsOpener() + cleanup() + dsInst, df = DsOpener() + wg.Add(Threads) drivers := make([]chan byte, Threads) for i := 0; i < Threads; i++ { drivers[i] = make(chan byte, 15) @@ -44,6 +51,13 @@ func setup() ([]chan byte, context.CancelFunc) { return drivers, cncl } +func cleanup() { + if dsInst != nil { + dsInst.Close() + df() + } +} + // Fuzz is a go-fuzzer compatible input point for replaying // data (interpreted as a script of commands) // to a chosen ipfs datastore implementation @@ -54,7 +68,8 @@ func Fuzz(data []byte) int { close(drivers[i]) } cncl() - dsInst.Close() + wg.Wait() + cleanup() return 0 } @@ -79,7 +94,8 @@ func FuzzStream(data io.Reader) int { close(drivers[i]) } cncl() - dsInst.Close() + wg.Wait() + cleanup() return 0 } @@ -111,6 +127,7 @@ type state struct { } func threadDriver(ctx context.Context, cmnds chan byte) error { + defer wg.Done() s := state{} s.reader = dsInst s.writer = dsInst diff --git a/go.mod b/go.mod index 4be9a76..adb3eff 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/ipfs/go-datastore require ( + github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813 // indirect github.com/google/uuid v1.1.1 github.com/ipfs/go-ds-badger v0.2.1 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8 diff --git a/go.sum b/go.sum index 7add6d6..fb21705 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813 h1:NgO45/5mBLRVfiXerEFzH6ikcZ7DNRPS639xFg3ENzU= +github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= From 3578d51729e4f698f7eab00eac5e59e4cd6e27ec Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 27 Mar 2020 10:29:11 -0700 Subject: [PATCH 03/12] fuzzer had bug on 0-length values --- fuzz/fuzzer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fuzz/fuzzer.go b/fuzz/fuzzer.go index 4cd5f04..48e7f00 100644 --- a/fuzz/fuzzer.go +++ b/fuzz/fuzzer.go @@ -262,7 +262,9 @@ func makeKey(s *state, c byte) error { func makeValue(s *state, c byte) error { s.val = make([]byte, c) - s.val[0] = 1 + if c != 0 { + s.val[0] = 1 + } s.valReady = true return nil } From 53074b5d20f56b449a2f66235132f7234ea11bf4 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 27 Mar 2020 13:26:19 -0700 Subject: [PATCH 04/12] add comparison for consistency between badger and level --- fuzz/cmd/compare/main.go | 100 +++++++++++++++++++++++++++++++++++++++ fuzz/cmd/run/main.go | 7 +-- fuzz/fuzzer.go | 8 ++-- go.mod | 1 + go.sum | 18 +++++++ 5 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 fuzz/cmd/compare/main.go diff --git a/fuzz/cmd/compare/main.go b/fuzz/cmd/compare/main.go new file mode 100644 index 0000000..68949e1 --- /dev/null +++ b/fuzz/cmd/compare/main.go @@ -0,0 +1,100 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + + ds "github.com/ipfs/go-datastore" + fuzzer "github.com/ipfs/go-datastore/fuzz" + dsq "github.com/ipfs/go-datastore/query" + badger "github.com/ipfs/go-ds-badger" + leveldb "github.com/ipfs/go-ds-leveldb" + + "github.com/spf13/pflag" +) + +var input *string = pflag.StringP("input", "i", "", "file to read input from (stdin used if not specified)") +var db1 *string = pflag.StringP("db1", "d", "badger", "database to fuzz") +var db2 *string = pflag.StringP("db2", "e", "level", "database to fuzz") +var dbFile *string = pflag.StringP("file", "f", "tmp", "where the db instaces should live on disk") +var threads *int = pflag.IntP("threads", "t", 1, "concurrent threads") + +func openDB(db string) { + if db == "badger" { + fuzzer.DsOpener = func() (ds.TxnDatastore, fuzzer.Donefunc) { + d, err := badger.NewDatastore(*dbFile, &badger.DefaultOptions) + if err != nil { + panic("could not create db instance") + } + donefunc := func() error { return nil } + return d, donefunc + } + } else if db == "level" { + fuzzer.DsOpener = func() (ds.TxnDatastore, fuzzer.Donefunc) { + d, err := leveldb.NewDatastore(*dbFile, &leveldb.Options{}) + if err != nil { + panic("could not create db instance") + } + donefunc := func() error { return nil } + return d, donefunc + } + } else { + // TODO + panic("unknown database") + } +} + +func main() { + pflag.Parse() + + // do one, then the other, then compare state. + + fuzzer.Threads = *threads + + var dat []byte + var err error + if *input == "" { + dat, err = ioutil.ReadAll(os.Stdin) + } else { + dat, err = ioutil.ReadFile(*input) + } + if err != nil { + fmt.Fprintf(os.Stderr, "could not read %s: %v\n", *input, err) + return + } + + base := *dbFile + *dbFile = base + "1" + openDB(*db1) + ret := fuzzer.Fuzz(dat) + + db1, _ := fuzzer.DsOpener() + + *dbFile = base + "2" + openDB(*db2) + ret = fuzzer.Fuzz(dat) + + db2, _ := fuzzer.DsOpener() + + // compare. + r1, err := db1.Query(dsq.Query{}) + if err != nil { + panic(err) + } + + for r := range r1.Next() { + if r.Error != nil { + // handle. + break + } + + if exist, _ := db2.Has(ds.NewKey(r.Entry.Key)); !exist { + fmt.Fprintf(os.Stderr, "db2 failed to get key %s held by db1\n", r.Entry.Key) + } + } + db1.Close() + db2.Close() + + os.Exit(ret) +} diff --git a/fuzz/cmd/run/main.go b/fuzz/cmd/run/main.go index 2c4dceb..81b6b87 100644 --- a/fuzz/cmd/run/main.go +++ b/fuzz/cmd/run/main.go @@ -7,8 +7,8 @@ import ( "os" ds "github.com/ipfs/go-datastore" - badger "github.com/ipfs/go-ds-badger" fuzzer "github.com/ipfs/go-datastore/fuzz" + badger "github.com/ipfs/go-ds-badger" "github.com/spf13/pflag" ) @@ -23,12 +23,13 @@ func main() { fuzzer.Threads = *threads if *db == "badger" { - fuzzer.DsOpener = func() ds.TxnDatastore { + fuzzer.DsOpener = func() (ds.TxnDatastore, fuzzer.Donefunc) { d, err := badger.NewDatastore(*dbFile, &badger.DefaultOptions) if err != nil { panic("could not create db instance") } - return d + donefunc := func() error { return nil } + return d, donefunc } } else { // TODO diff --git a/fuzz/fuzzer.go b/fuzz/fuzzer.go index 48e7f00..80d919f 100644 --- a/fuzz/fuzzer.go +++ b/fuzz/fuzzer.go @@ -12,19 +12,19 @@ import ( badger "github.com/ipfs/go-ds-badger" ) -type donefunc func() error +type Donefunc func() error // DsOpener is the concrete datastore. that Fuzz will fuzz against. -var DsOpener func() (ds.TxnDatastore, donefunc) +var DsOpener func() (ds.TxnDatastore, Donefunc) var dsInst ds.TxnDatastore -var df donefunc +var df Donefunc // Threads is a measure of concurrency. var Threads int var wg sync.WaitGroup func init() { - DsOpener = func() (ds.TxnDatastore, donefunc) { + DsOpener = func() (ds.TxnDatastore, Donefunc) { dir, _ := ioutil.TempDir("", "fuzz*") d, _ := badger.NewDatastore(dir, &badger.DefaultOptions) return d, func() error { return os.RemoveAll(dir) } diff --git a/go.mod b/go.mod index adb3eff..dc4b59d 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ require ( github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813 // indirect github.com/google/uuid v1.1.1 github.com/ipfs/go-ds-badger v0.2.1 + github.com/ipfs/go-ds-leveldb v0.4.2 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8 github.com/jbenet/goprocess v0.1.3 github.com/spf13/pflag v1.0.3 diff --git a/go.sum b/go.sum index fb21705..408ce6b 100644 --- a/go.sum +++ b/go.sum @@ -19,15 +19,22 @@ github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-ds-badger v0.2.1 h1:RsC9DDlwFhFdfT+s2PeC8joxbSp2YMufK8w/RBOxKtk= github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= +github.com/ipfs/go-ds-leveldb v0.4.2 h1:QmQoAJ9WkPMUfBLnu1sBVy0xWWlJPg0m4kRAiJL9iaw= +github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8 h1:NAviDvJ0WXgD+yiL2Rj35AmnfgI11+pHXbdciD917U0= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= @@ -52,6 +59,9 @@ github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -69,15 +79,20 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -90,4 +105,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 2fafe54420581eeba1e9e261c62de78139b13760 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 27 Mar 2020 13:51:05 -0700 Subject: [PATCH 05/12] comparison --- fuzz/cmd/compare/main.go | 32 +++++++++++++++++++++++++++++++- fuzz/fuzzer.go | 21 +++++++++++++++++++-- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/fuzz/cmd/compare/main.go b/fuzz/cmd/compare/main.go index 68949e1..0d33ea3 100644 --- a/fuzz/cmd/compare/main.go +++ b/fuzz/cmd/compare/main.go @@ -64,37 +64,67 @@ func main() { return } + fmt.Printf("running db 1\n") base := *dbFile *dbFile = base + "1" + fuzzer.RandSeed(0) openDB(*db1) ret := fuzzer.Fuzz(dat) db1, _ := fuzzer.DsOpener() + fmt.Printf("running db 2\n") *dbFile = base + "2" + fuzzer.RandSeed(0) openDB(*db2) ret = fuzzer.Fuzz(dat) db2, _ := fuzzer.DsOpener() - + fmt.Printf("done\n") // compare. r1, err := db1.Query(dsq.Query{}) if err != nil { panic(err) } + fmt.Printf("checking equality\n") for r := range r1.Next() { if r.Error != nil { // handle. break } + if r.Entry.Key == "/" { + continue + } if exist, _ := db2.Has(ds.NewKey(r.Entry.Key)); !exist { fmt.Fprintf(os.Stderr, "db2 failed to get key %s held by db1\n", r.Entry.Key) } } + + r2, err := db2.Query(dsq.Query{}) + if err != nil { + panic(err) + } + + fmt.Printf("checking equality\n") + for r := range r2.Next() { + if r.Error != nil { + // handle. + break + } + if r.Entry.Key == "/" { + continue + } + + if exist, _ := db1.Has(ds.NewKey(r.Entry.Key)); !exist { + fmt.Fprintf(os.Stderr, "db1 failed to get key %s held by db2\n", r.Entry.Key) + } + } + db1.Close() db2.Close() + fmt.Printf("done\n") os.Exit(ret) } diff --git a/fuzz/fuzzer.go b/fuzz/fuzzer.go index 80d919f..5853b26 100644 --- a/fuzz/fuzzer.go +++ b/fuzz/fuzzer.go @@ -2,6 +2,7 @@ package fuzzer import ( "context" + "fmt" "io" "io/ioutil" "os" @@ -9,6 +10,7 @@ import ( "sync/atomic" ds "github.com/ipfs/go-datastore" + dsq "github.com/ipfs/go-datastore/query" badger "github.com/ipfs/go-ds-badger" ) @@ -19,6 +21,12 @@ var DsOpener func() (ds.TxnDatastore, Donefunc) var dsInst ds.TxnDatastore var df Donefunc +var ctr int32 + +func RandSeed(seed int32) { + ctr = seed +} + // Threads is a measure of concurrency. var Threads int var wg sync.WaitGroup @@ -171,7 +179,16 @@ func nextState(s *state, c byte) error { reset(s) return nil } else if s.op == opQuery { - // TODO + r, _ := s.reader.Query(dsq.Query{}) + defer r.Close() + reset(s) + + for e := range r.Next() { + if e.Error != nil { + return nil + } + } + return nil } else if s.op == opPut { if !s.keyReady { return makeKey(s, c) @@ -241,7 +258,7 @@ func makeKey(s *state, c byte) error { s.key = keyCache[(c>>1)%byte(keys)] s.keyReady = true } else { - s.key = ds.RandomKey() + s.key = ds.NewKey(fmt.Sprintf("key-%d", atomic.AddInt32(&ctr, 1))) // half the time we'll make it a child of an existing key if c&2 == 2 { s.key = keyCache[(c>>1)%byte(keys)].Child(s.key) From 5697c8dc09ec2d4cad81ac627f2f7dab8826df4c Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 27 Mar 2020 13:55:10 -0700 Subject: [PATCH 06/12] reset other point of non-determinism --- fuzz/fuzzer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fuzz/fuzzer.go b/fuzz/fuzzer.go index 5853b26..c0afe2a 100644 --- a/fuzz/fuzzer.go +++ b/fuzz/fuzzer.go @@ -25,6 +25,8 @@ var ctr int32 func RandSeed(seed int32) { ctr = seed + // also reset the key cache. + cachedKeys = 1 } // Threads is a measure of concurrency. From 7fa0f19f266a98d96c138cbe0d44b1fe47fcadb7 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 27 Mar 2020 17:23:05 -0700 Subject: [PATCH 07/12] add prefix checker for validating a partial run --- fuzz/cmd/compare/main.go | 31 +---------- fuzz/cmd/isprefix/main.go | 111 ++++++++++++++++++++++++++++++++++++++ fuzz/cmd/run/main.go | 17 +----- fuzz/fuzzer.go | 55 +++++++++++++++---- 4 files changed, 161 insertions(+), 53 deletions(-) create mode 100644 fuzz/cmd/isprefix/main.go diff --git a/fuzz/cmd/compare/main.go b/fuzz/cmd/compare/main.go index 0d33ea3..c9731ee 100644 --- a/fuzz/cmd/compare/main.go +++ b/fuzz/cmd/compare/main.go @@ -8,8 +8,6 @@ import ( ds "github.com/ipfs/go-datastore" fuzzer "github.com/ipfs/go-datastore/fuzz" dsq "github.com/ipfs/go-datastore/query" - badger "github.com/ipfs/go-ds-badger" - leveldb "github.com/ipfs/go-ds-leveldb" "github.com/spf13/pflag" ) @@ -20,31 +18,6 @@ var db2 *string = pflag.StringP("db2", "e", "level", "database to fuzz") var dbFile *string = pflag.StringP("file", "f", "tmp", "where the db instaces should live on disk") var threads *int = pflag.IntP("threads", "t", 1, "concurrent threads") -func openDB(db string) { - if db == "badger" { - fuzzer.DsOpener = func() (ds.TxnDatastore, fuzzer.Donefunc) { - d, err := badger.NewDatastore(*dbFile, &badger.DefaultOptions) - if err != nil { - panic("could not create db instance") - } - donefunc := func() error { return nil } - return d, donefunc - } - } else if db == "level" { - fuzzer.DsOpener = func() (ds.TxnDatastore, fuzzer.Donefunc) { - d, err := leveldb.NewDatastore(*dbFile, &leveldb.Options{}) - if err != nil { - panic("could not create db instance") - } - donefunc := func() error { return nil } - return d, donefunc - } - } else { - // TODO - panic("unknown database") - } -} - func main() { pflag.Parse() @@ -68,7 +41,7 @@ func main() { base := *dbFile *dbFile = base + "1" fuzzer.RandSeed(0) - openDB(*db1) + fuzzer.SetOpener(*db1, *dbFile, false) ret := fuzzer.Fuzz(dat) db1, _ := fuzzer.DsOpener() @@ -76,7 +49,7 @@ func main() { fmt.Printf("running db 2\n") *dbFile = base + "2" fuzzer.RandSeed(0) - openDB(*db2) + fuzzer.SetOpener(*db2, *dbFile, false) ret = fuzzer.Fuzz(dat) db2, _ := fuzzer.DsOpener() diff --git a/fuzz/cmd/isprefix/main.go b/fuzz/cmd/isprefix/main.go new file mode 100644 index 0000000..25beef8 --- /dev/null +++ b/fuzz/cmd/isprefix/main.go @@ -0,0 +1,111 @@ +package main + +// sees if a db instance is equivalent to some prefix of running an input script. + +import ( + "fmt" + "io/ioutil" + "os" + + ds "github.com/ipfs/go-datastore" + fuzzer "github.com/ipfs/go-datastore/fuzz" + dsq "github.com/ipfs/go-datastore/query" + + "github.com/spf13/pflag" +) + +var input *string = pflag.StringP("input", "i", "", "file to read input from (stdin used if not specified)") +var db *string = pflag.StringP("db", "d", "badger", "database driver") +var dbPrev *string = pflag.StringP("exist", "e", "tmp1", "database instance already made") +var dbFile *string = pflag.StringP("file", "f", "tmp2", "where the replay should live") +var threads *int = pflag.IntP("threads", "t", 1, "concurrent threads") + +type validatingReader struct { + b []byte + i int + validator func(bool) bool + validI int +} + +func (v *validatingReader) Read(buf []byte) (n int, err error) { + if v.i == len(v.b) { + return 0, nil + } else { + if v.validator(false) { + v.validI = v.i + } + buf[0] = v.b[v.i] + v.i++ + return 1, nil + } +} + +func main() { + pflag.Parse() + + fuzzer.Threads = *threads + + var dat []byte + var err error + if *input == "" { + dat, err = ioutil.ReadAll(os.Stdin) + } else { + dat, err = ioutil.ReadFile(*input) + } + if err != nil { + fmt.Fprintf(os.Stderr, "could not read %s: %v\n", *input, err) + return + } + + fuzzer.SetOpener(*db, *dbPrev, false) + db1, _ := fuzzer.DsOpener() + + fuzzer.SetOpener(*db, *dbFile, false) + + fuzzer.RandSeed(0) + + reader := validatingReader{dat, 0, func(verbose bool) bool { + res, _ := fuzzer.GetInst().Query(dsq.Query{}) + for e := range res.Next() { + if e.Entry.Key == "/" { + continue + } + if h, _ := db1.Has(ds.NewKey(e.Entry.Key)); !h { + if verbose { + fmt.Printf("failed - script run db has %s not in existing.\n", e.Entry.Key) + } + return false // not yet complete + } + } + // next; make sure the other way is equal. + res, _ = db1.Query(dsq.Query{}) + for e := range res.Next() { + if e.Entry.Key == "/" { + continue + } + if h, _ := fuzzer.GetInst().Has(ds.NewKey(e.Entry.Key)); !h { + if verbose { + fmt.Printf("failed - existing db has %s not in replay.\n", e.Entry.Key) + } + return false + } + } + // db images are the same. + return true + }, -1} + fuzzer.FuzzStream(&reader) + if reader.validator(true) { + reader.validI = reader.i + } + fuzzer.Cleanup() + + db1.Close() + + if reader.validI > -1 { + fmt.Printf("Matched at stream position %d.\n", reader.validI) + os.Exit(0) + } else { + fmt.Printf("Failed to match\n") + os.Exit(1) + } +} diff --git a/fuzz/cmd/run/main.go b/fuzz/cmd/run/main.go index 81b6b87..45fa4cd 100644 --- a/fuzz/cmd/run/main.go +++ b/fuzz/cmd/run/main.go @@ -6,9 +6,7 @@ import ( "io/ioutil" "os" - ds "github.com/ipfs/go-datastore" fuzzer "github.com/ipfs/go-datastore/fuzz" - badger "github.com/ipfs/go-ds-badger" "github.com/spf13/pflag" ) @@ -22,19 +20,7 @@ func main() { pflag.Parse() fuzzer.Threads = *threads - if *db == "badger" { - fuzzer.DsOpener = func() (ds.TxnDatastore, fuzzer.Donefunc) { - d, err := badger.NewDatastore(*dbFile, &badger.DefaultOptions) - if err != nil { - panic("could not create db instance") - } - donefunc := func() error { return nil } - return d, donefunc - } - } else { - // TODO - panic("unknown database") - } + fuzzer.SetOpener(*db, *dbFile, false) if *input != "" { dat, err := ioutil.ReadFile(*input) @@ -46,6 +32,7 @@ func main() { os.Exit(ret) } else { ret := fuzzer.FuzzStream(bufio.NewReader(os.Stdin)) + fuzzer.Cleanup() os.Exit(ret) } } diff --git a/fuzz/fuzzer.go b/fuzz/fuzzer.go index c0afe2a..1854de4 100644 --- a/fuzz/fuzzer.go +++ b/fuzz/fuzzer.go @@ -12,6 +12,7 @@ import ( ds "github.com/ipfs/go-datastore" dsq "github.com/ipfs/go-datastore/query" badger "github.com/ipfs/go-ds-badger" + leveldb "github.com/ipfs/go-ds-leveldb" ) type Donefunc func() error @@ -34,22 +35,24 @@ var Threads int var wg sync.WaitGroup func init() { - DsOpener = func() (ds.TxnDatastore, Donefunc) { - dir, _ := ioutil.TempDir("", "fuzz*") - d, _ := badger.NewDatastore(dir, &badger.DefaultOptions) - return d, func() error { return os.RemoveAll(dir) } - } + dir, _ := ioutil.TempDir("", "fuzz*") + SetOpener("badger", dir, true) Threads = 1 keyCache[0] = ds.NewKey("/") cachedKeys = 1 } +// GetInst gets the current DB handle. +func GetInst() ds.TxnDatastore { + return dsInst +} + func setup() ([]chan byte, context.CancelFunc) { // TODO: dynamic thread starting. ctx, cncl := context.WithCancel(context.Background()) - cleanup() + Cleanup() dsInst, df = DsOpener() wg.Add(Threads) @@ -61,7 +64,7 @@ func setup() ([]chan byte, context.CancelFunc) { return drivers, cncl } -func cleanup() { +func Cleanup() { if dsInst != nil { dsInst.Close() df() @@ -79,7 +82,7 @@ func Fuzz(data []byte) int { } cncl() wg.Wait() - cleanup() + Cleanup() return 0 } @@ -105,7 +108,6 @@ func FuzzStream(data io.Reader) int { } cncl() wg.Wait() - cleanup() return 0 } @@ -122,6 +124,7 @@ const ( opNewTX opCommitTX opDiscardTX + opSync opMax ) @@ -236,6 +239,13 @@ func nextState(s *state, c byte) error { } reset(s) return nil + } else if s.op == opSync { + if !s.keyReady { + return makeKey(s, c) + } + dsInst.Sync(s.key) + reset(s) + return nil } return nil } @@ -287,3 +297,30 @@ func makeValue(s *state, c byte) error { s.valReady = true return nil } + +func SetOpener(driver string, loc string, cleanup bool) { + donefunc := func() error { return nil } + if cleanup { + donefunc = func() error { return os.RemoveAll(loc) } + } + if driver == "badger" { + DsOpener = func() (ds.TxnDatastore, Donefunc) { + d, err := badger.NewDatastore(loc, &badger.DefaultOptions) + if err != nil { + panic("could not create db instance") + } + return d, donefunc + } + } else if driver == "level" { + DsOpener = func() (ds.TxnDatastore, Donefunc) { + d, err := leveldb.NewDatastore(loc, &leveldb.Options{}) + if err != nil { + panic("could not create db instance") + } + return d, donefunc + } + } else { + // TODO + panic("unknown database") + } +} From c333b9d24e30ef2728649d98dc4c178648d2b242 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 2 Apr 2020 20:51:27 -0700 Subject: [PATCH 08/12] Clean up fuzzing and allow fuzzing against injected datastores --- fuzz/.gitignore | 5 + fuzz/README.md | 25 +++- fuzz/cmd/compare/main.go | 53 ++++---- fuzz/cmd/generate/main.go | 68 ++++++++++ fuzz/cmd/isprefix/main.go | 48 +++---- fuzz/cmd/run/main.go | 17 ++- fuzz/fuzzer.go | 262 +++++++++++++++++++++----------------- 7 files changed, 304 insertions(+), 174 deletions(-) create mode 100644 fuzz/.gitignore create mode 100644 fuzz/cmd/generate/main.go diff --git a/fuzz/.gitignore b/fuzz/.gitignore new file mode 100644 index 0000000..ae1c595 --- /dev/null +++ b/fuzz/.gitignore @@ -0,0 +1,5 @@ +provider_*.go +*.zip +corpus +crashers +suppressions diff --git a/fuzz/README.md b/fuzz/README.md index 686e549..39ac629 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -2,5 +2,28 @@ IPFS Datastore Fuzzer ==== The fuzzer provides a [go fuzzer](https://github.com/dvyukov/go-fuzz) interface -to the Datastore interface. This can be used for fuzz testing of datastore +to Datastore implementations. This can be used for fuzz testing of these implementations. + +Usage +---- + +First, get the code +```golang +go get github.com/ipfs/go-datastore +cd go-datastore/fuzz +``` + +Next, configure the datastores to fuzz (from this directory) +```golang +// either run via `go run` +go run ./cmd/generate github.com/ipfs/go-ds-badger +// or `go generate` +DS_PROVIDERS="github.com/ipfs/go-ds-badger" go generate +``` + +Finally, build the fuzzing artifact, and fuzz +```golang +go-fuzz-build +go-fuzz +``` diff --git a/fuzz/cmd/compare/main.go b/fuzz/cmd/compare/main.go index c9731ee..be78146 100644 --- a/fuzz/cmd/compare/main.go +++ b/fuzz/cmd/compare/main.go @@ -15,7 +15,7 @@ import ( var input *string = pflag.StringP("input", "i", "", "file to read input from (stdin used if not specified)") var db1 *string = pflag.StringP("db1", "d", "badger", "database to fuzz") var db2 *string = pflag.StringP("db2", "e", "level", "database to fuzz") -var dbFile *string = pflag.StringP("file", "f", "tmp", "where the db instaces should live on disk") +var dbFile *string = pflag.StringP("file", "f", "tmp", "where the db instances should live on disk") var threads *int = pflag.IntP("threads", "t", 1, "concurrent threads") func main() { @@ -33,37 +33,44 @@ func main() { dat, err = ioutil.ReadFile(*input) } if err != nil { - fmt.Fprintf(os.Stderr, "could not read %s: %v\n", *input, err) + fmt.Fprintf(os.Stderr, "Could not read %s: %v\n", *input, err) return } - fmt.Printf("running db 1\n") - base := *dbFile - *dbFile = base + "1" - fuzzer.RandSeed(0) - fuzzer.SetOpener(*db1, *dbFile, false) - ret := fuzzer.Fuzz(dat) - - db1, _ := fuzzer.DsOpener() + db1loc := *dbFile + "1" + inst1, err := fuzzer.Open(*db1, db1loc, false) + if err != nil { + fmt.Fprintf(os.Stderr, "Could not open db: %v\n", err) + return + } + defer inst1.Cancel() - fmt.Printf("running db 2\n") - *dbFile = base + "2" - fuzzer.RandSeed(0) - fuzzer.SetOpener(*db2, *dbFile, false) - ret = fuzzer.Fuzz(dat) + db2loc := *dbFile + "2" + inst2, err := fuzzer.Open(*db2, db2loc, false) + if err != nil { + inst1.Cancel() + fmt.Fprintf(os.Stderr, "Could not open db: %v\n", err) + return + } + defer inst2.Cancel() - db2, _ := fuzzer.DsOpener() + fmt.Printf("Running db1.........") + inst1.Fuzz(dat) + fmt.Printf("done\n") + fmt.Printf("Running db2.........") + inst2.Fuzz(dat) fmt.Printf("done\n") - // compare. + + fmt.Printf("Checking equality...") + db1 := inst1.DB() + db2 := inst2.DB() r1, err := db1.Query(dsq.Query{}) if err != nil { panic(err) } - fmt.Printf("checking equality\n") for r := range r1.Next() { if r.Error != nil { - // handle. break } if r.Entry.Key == "/" { @@ -80,10 +87,8 @@ func main() { panic(err) } - fmt.Printf("checking equality\n") for r := range r2.Next() { if r.Error != nil { - // handle. break } if r.Entry.Key == "/" { @@ -95,9 +100,7 @@ func main() { } } - db1.Close() - db2.Close() - fmt.Printf("done\n") + fmt.Printf("Done\n") - os.Exit(ret) + return } diff --git a/fuzz/cmd/generate/main.go b/fuzz/cmd/generate/main.go new file mode 100644 index 0000000..84d64c3 --- /dev/null +++ b/fuzz/cmd/generate/main.go @@ -0,0 +1,68 @@ +// This file is invoked at the fuzzer level by `go generate` +package main + +import ( + "fmt" + "os" + "os/exec" + "strings" + "text/template" +) + +// This program generates bindings to fuzz a concrete datastore implementation. +// It can be invoked by running `go generate ` + +func main() { + providers := os.Args[1:] + + if len(providers) == 0 { + providers = strings.Split(os.Getenv("DS_PROVIDERS"), ",") + } + if len(providers) == 0 { + fmt.Fprintf(os.Stderr, "No providers specified to generate. Nothing to do.") + return + } + + for _, provider := range providers { + provider = strings.TrimSpace(provider) + cmd := exec.Command("go", "get", provider) + err := cmd.Run() + if err != nil { + fmt.Fprintf(os.Stderr, "failed to add dependency for %s: %v\n", provider, err) + os.Exit(1) + } + + nameComponents := strings.Split(provider, "/") + name := nameComponents[len(nameComponents)-1] + f, err := os.Create(fmt.Sprintf("provider_%s.go", name)) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to create provider file: %v", err) + os.Exit(1) + } + defer f.Close() + provideTemplate.Execute(f, struct { + Package string + PackageName string + }{ + Package: provider, + PackageName: name, + }) + } +} + +var provideTemplate = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT. + +package fuzzer +import prov "{{ .Package }}" +import ds "github.com/ipfs/go-datastore" + +func init() { + AddOpener("{{ .PackageName }}", func(loc string) ds.TxnDatastore { + d, err := prov.NewDatastore(loc, nil) + if err != nil { + panic("could not create db instance") + } + return d + }) +} +`)) diff --git a/fuzz/cmd/isprefix/main.go b/fuzz/cmd/isprefix/main.go index 25beef8..a6f7555 100644 --- a/fuzz/cmd/isprefix/main.go +++ b/fuzz/cmd/isprefix/main.go @@ -1,6 +1,6 @@ package main -// sees if a db instance is equivalent to some prefix of running an input script. +// Checks if a db instance is equivalent to some prefix of an input script. import ( "fmt" @@ -15,7 +15,7 @@ import ( ) var input *string = pflag.StringP("input", "i", "", "file to read input from (stdin used if not specified)") -var db *string = pflag.StringP("db", "d", "badger", "database driver") +var db *string = pflag.StringP("db", "d", "go-ds-badger", "database driver") var dbPrev *string = pflag.StringP("exist", "e", "tmp1", "database instance already made") var dbFile *string = pflag.StringP("file", "f", "tmp2", "where the replay should live") var threads *int = pflag.IntP("threads", "t", 1, "concurrent threads") @@ -30,14 +30,13 @@ type validatingReader struct { func (v *validatingReader) Read(buf []byte) (n int, err error) { if v.i == len(v.b) { return 0, nil - } else { - if v.validator(false) { - v.validI = v.i - } - buf[0] = v.b[v.i] - v.i++ - return 1, nil } + if v.validator(false) { + v.validI = v.i + } + buf[0] = v.b[v.i] + v.i++ + return 1, nil } func main() { @@ -57,20 +56,27 @@ func main() { return } - fuzzer.SetOpener(*db, *dbPrev, false) - db1, _ := fuzzer.DsOpener() - - fuzzer.SetOpener(*db, *dbFile, false) + previousDB, err := fuzzer.Open(*db, *dbPrev, false) + if err != nil { + fmt.Fprintf(os.Stderr, "could not open: %v\n", err) + return + } + defer previousDB.Cancel() - fuzzer.RandSeed(0) + replayDB, err := fuzzer.Open(*db, *dbFile, true) + if err != nil { + fmt.Fprintf(os.Stderr, "could not open: %v\n", err) + return + } + defer replayDB.Cancel() reader := validatingReader{dat, 0, func(verbose bool) bool { - res, _ := fuzzer.GetInst().Query(dsq.Query{}) + res, _ := replayDB.DB().Query(dsq.Query{}) for e := range res.Next() { if e.Entry.Key == "/" { continue } - if h, _ := db1.Has(ds.NewKey(e.Entry.Key)); !h { + if h, _ := previousDB.DB().Has(ds.NewKey(e.Entry.Key)); !h { if verbose { fmt.Printf("failed - script run db has %s not in existing.\n", e.Entry.Key) } @@ -78,12 +84,12 @@ func main() { } } // next; make sure the other way is equal. - res, _ = db1.Query(dsq.Query{}) + res, _ = previousDB.DB().Query(dsq.Query{}) for e := range res.Next() { if e.Entry.Key == "/" { continue } - if h, _ := fuzzer.GetInst().Has(ds.NewKey(e.Entry.Key)); !h { + if h, _ := replayDB.DB().Has(ds.NewKey(e.Entry.Key)); !h { if verbose { fmt.Printf("failed - existing db has %s not in replay.\n", e.Entry.Key) } @@ -93,13 +99,11 @@ func main() { // db images are the same. return true }, -1} - fuzzer.FuzzStream(&reader) + + replayDB.FuzzStream(&reader) if reader.validator(true) { reader.validI = reader.i } - fuzzer.Cleanup() - - db1.Close() if reader.validI > -1 { fmt.Printf("Matched at stream position %d.\n", reader.validI) diff --git a/fuzz/cmd/run/main.go b/fuzz/cmd/run/main.go index 45fa4cd..332a544 100644 --- a/fuzz/cmd/run/main.go +++ b/fuzz/cmd/run/main.go @@ -12,7 +12,7 @@ import ( ) var input *string = pflag.StringP("input", "i", "", "file to read input from (stdin used if not specified)") -var db *string = pflag.StringP("database", "d", "badger", "database to fuzz") +var db *string = pflag.StringP("database", "d", "go-ds-badger", "database to fuzz") var dbFile *string = pflag.StringP("file", "f", "tmp", "where the db instace should live on disk") var threads *int = pflag.IntP("threads", "t", 1, "concurrent threads") @@ -20,19 +20,22 @@ func main() { pflag.Parse() fuzzer.Threads = *threads - fuzzer.SetOpener(*db, *dbFile, false) if *input != "" { dat, err := ioutil.ReadFile(*input) if err != nil { fmt.Fprintf(os.Stderr, "could not read %s: %v\n", *input, err) - return + os.Exit(1) } - ret := fuzzer.Fuzz(dat) + ret := fuzzer.FuzzDB(*db, *dbFile, false, dat) os.Exit(ret) } else { - ret := fuzzer.FuzzStream(bufio.NewReader(os.Stdin)) - fuzzer.Cleanup() - os.Exit(ret) + reader := bufio.NewReader(os.Stdin) + err := fuzzer.FuzzStream(*db, *dbFile, false, reader) + if err != nil { + fmt.Fprintf(os.Stderr, "Error fuzzing: %v\n", err) + os.Exit(1) + } + return } } diff --git a/fuzz/fuzzer.go b/fuzz/fuzzer.go index 1854de4..b574a8f 100644 --- a/fuzz/fuzzer.go +++ b/fuzz/fuzzer.go @@ -11,104 +11,168 @@ import ( ds "github.com/ipfs/go-datastore" dsq "github.com/ipfs/go-datastore/query" - badger "github.com/ipfs/go-ds-badger" - leveldb "github.com/ipfs/go-ds-leveldb" ) -type Donefunc func() error +//go:generate go run ./cmd/generate -// DsOpener is the concrete datastore. that Fuzz will fuzz against. -var DsOpener func() (ds.TxnDatastore, Donefunc) -var dsInst ds.TxnDatastore -var df Donefunc +// openers contains the known datastore implementations. +var openers map[string]func(string) ds.TxnDatastore -var ctr int32 - -func RandSeed(seed int32) { - ctr = seed - // also reset the key cache. - cachedKeys = 1 +// AddOpener allows registration of a new driver for fuzzing. +func AddOpener(name string, opener func(loc string) ds.TxnDatastore) { + if openers == nil { + openers = make(map[string]func(string) ds.TxnDatastore) + } + openers[name] = opener } // Threads is a measure of concurrency. +// Note: if Threads > 1, determinism is not guaranteed. var Threads int -var wg sync.WaitGroup func init() { - dir, _ := ioutil.TempDir("", "fuzz*") - SetOpener("badger", dir, true) + if openers == nil { + openers = make(map[string]func(string) ds.TxnDatastore) + } Threads = 1 +} - keyCache[0] = ds.NewKey("/") - cachedKeys = 1 +// RunState encapulates the state of a given fuzzing run +type RunState struct { + inst ds.TxnDatastore + inputChannels []chan<- byte + wg sync.WaitGroup + Cancel context.CancelFunc + + keyCache [128]ds.Key + cachedKeys int32 + ctr int32 +} + +// DB returns the datastore being driven by this instance +func (r *RunState) DB() ds.TxnDatastore { + return r.inst } -// GetInst gets the current DB handle. -func GetInst() ds.TxnDatastore { - return dsInst +type threadState struct { + op + keyReady bool + key ds.Key + valReady bool + val []byte + reader ds.Read + writer ds.Write + txn ds.Txn + *RunState } -func setup() ([]chan byte, context.CancelFunc) { - // TODO: dynamic thread starting. +// Open instantiates an instance of the database implementation for testing. +func Open(driver string, location string, cleanup bool) (*RunState, error) { ctx, cncl := context.WithCancel(context.Background()) - Cleanup() - dsInst, df = DsOpener() + opener, ok := openers[driver] + if !ok { + cncl() + return nil, fmt.Errorf("no such driver: %s", driver) + } - wg.Add(Threads) - drivers := make([]chan byte, Threads) - for i := 0; i < Threads; i++ { - drivers[i] = make(chan byte, 15) - go threadDriver(ctx, drivers[i]) + state := RunState{} + state.inst = opener(location) + state.keyCache[0] = ds.NewKey("/") + state.cachedKeys = 1 + + state.wg.Add(Threads) + + // wrap the context cancel to block until everythign is fully closed. + doneCh := make(chan struct{}) + state.Cancel = func() { + for i := 0; i < Threads; i++ { + close(state.inputChannels[i]) + } + cncl() + <-doneCh } - return drivers, cncl -} + go func() { + state.wg.Wait() + state.inst.Close() + if cleanup { + os.RemoveAll(location) + } + close(doneCh) + }() -func Cleanup() { - if dsInst != nil { - dsInst.Close() - df() + state.inputChannels = make([]chan<- byte, Threads) + for i := 0; i < Threads; i++ { + dr := make(chan byte, 15) + go threadDriver(ctx, &state, dr) + state.inputChannels[i] = dr } + return &state, nil } // Fuzz is a go-fuzzer compatible input point for replaying // data (interpreted as a script of commands) -// to a chosen ipfs datastore implementation +// to kown ipfs datastore implementations func Fuzz(data []byte) int { - drivers, cncl := setup() - drive(drivers, data) - for i := 0; i < Threads; i++ { - close(drivers[i]) + var impls []string + for impl := range openers { + impls = append(impls, impl) + } + + defaultLoc, _ := ioutil.TempDir("", "fuzz-*") + + if len(impls) == 0 { + fmt.Fprintf(os.Stderr, "No datastores to fuzz.\n") + return -1 + } else if len(impls) == 1 { + return FuzzDB(impls[0], defaultLoc, true, data) + } else { + impl := impls[int(data[0])%len(impls)] + return FuzzDB(impl, defaultLoc, true, data[1:]) + } +} + +// FuzzDB fuzzes a given database entry, providing sufficient hooks to be +// used by CLI commands. +func FuzzDB(driver string, location string, cleanup bool, data []byte) int { + inst, err := Open(driver, location, cleanup) + if err != nil { + return -1 } - cncl() - wg.Wait() - Cleanup() + inst.Fuzz(data) + inst.Cancel() return 0 } -func drive(drivers []chan byte, data []byte) { +// FuzzStream does the same as fuzz but with streaming input +func FuzzStream(driver string, location string, cleanup bool, data io.Reader) error { + inst, err := Open("badger", "tmp", true) + if err != nil { + return err + } + + inst.FuzzStream(data) + inst.Cancel() + return nil +} + +// Fuzz sends a set of bytes to drive the current open datastore instance. +func (r *RunState) Fuzz(data []byte) { for i, b := range data { - drivers[i%Threads] <- b + r.inputChannels[i%Threads] <- b } } -// FuzzStream does the same as fuzz but with streaming input -func FuzzStream(data io.Reader) int { - drivers, cncl := setup() +// FuzzStream sends a set of bytes to drive the current instance from a reader. +func (r *RunState) FuzzStream(data io.Reader) { b := make([]byte, 4096) for { n, _ := data.Read(b) if n == 0 { break } - drive(drivers, b[:n]) + r.Fuzz(b[:n]) } - for i := 0; i < Threads; i++ { - close(drivers[i]) - } - cncl() - wg.Wait() - return 0 } type op byte @@ -128,22 +192,12 @@ const ( opMax ) -type state struct { - op - keyReady bool - key ds.Key - valReady bool - val []byte - reader ds.Read - writer ds.Write - txn ds.Txn -} - -func threadDriver(ctx context.Context, cmnds chan byte) error { - defer wg.Done() - s := state{} - s.reader = dsInst - s.writer = dsInst +func threadDriver(ctx context.Context, runState *RunState, cmnds chan byte) error { + defer runState.wg.Done() + s := threadState{} + s.RunState = runState + s.reader = runState.inst + s.writer = runState.inst for { select { @@ -158,7 +212,7 @@ func threadDriver(ctx context.Context, cmnds chan byte) error { } } -func nextState(s *state, c byte) error { +func nextState(s *threadState, c byte) error { if s.op == opNone { s.op = op(c) % opMax return nil @@ -213,7 +267,7 @@ func nextState(s *state, c byte) error { return nil } else if s.op == opNewTX { if s.txn == nil { - s.txn, _ = dsInst.NewTransaction(((c & 1) == 1)) + s.txn, _ = s.RunState.inst.NewTransaction(((c & 1) == 1)) if (c & 1) != 1 { // read+write s.writer = s.txn } @@ -225,8 +279,8 @@ func nextState(s *state, c byte) error { if s.txn != nil { s.txn.Discard() s.txn = nil - s.reader = dsInst - s.writer = dsInst + s.reader = s.RunState.inst + s.writer = s.RunState.inst } reset(s) return nil @@ -234,8 +288,8 @@ func nextState(s *state, c byte) error { if s.txn != nil { s.txn.Discard() s.txn = nil - s.reader = dsInst - s.writer = dsInst + s.reader = s.RunState.inst + s.writer = s.RunState.inst } reset(s) return nil @@ -243,45 +297,42 @@ func nextState(s *state, c byte) error { if !s.keyReady { return makeKey(s, c) } - dsInst.Sync(s.key) + s.RunState.inst.Sync(s.key) reset(s) return nil } return nil } -func reset(s *state) { +func reset(s *threadState) { s.op = opNone s.keyReady = false s.key = ds.RawKey("") s.valReady = false } -var keyCache [128]ds.Key -var cachedKeys int32 - -func makeKey(s *state, c byte) error { - keys := atomic.LoadInt32(&cachedKeys) +func makeKey(s *threadState, c byte) error { + keys := atomic.LoadInt32(&s.RunState.cachedKeys) if keys > 128 { keys = 128 } if c&1 == 1 { // 50% chance we want to-reuse an existing key - s.key = keyCache[(c>>1)%byte(keys)] + s.key = s.RunState.keyCache[(c>>1)%byte(keys)] s.keyReady = true } else { - s.key = ds.NewKey(fmt.Sprintf("key-%d", atomic.AddInt32(&ctr, 1))) + s.key = ds.NewKey(fmt.Sprintf("key-%d", atomic.AddInt32(&s.ctr, 1))) // half the time we'll make it a child of an existing key if c&2 == 2 { - s.key = keyCache[(c>>1)%byte(keys)].Child(s.key) + s.key = s.RunState.keyCache[(c>>1)%byte(keys)].Child(s.key) } // new key if keys < 128 { - keys = atomic.AddInt32(&cachedKeys, 1) + keys = atomic.AddInt32(&s.RunState.cachedKeys, 1) if keys >= 128 { - atomic.StoreInt32(&cachedKeys, 128) + atomic.StoreInt32(&s.RunState.cachedKeys, 128) } else { - keyCache[keys-1] = s.key + s.RunState.keyCache[keys-1] = s.key } } s.keyReady = true @@ -289,7 +340,7 @@ func makeKey(s *state, c byte) error { return nil } -func makeValue(s *state, c byte) error { +func makeValue(s *threadState, c byte) error { s.val = make([]byte, c) if c != 0 { s.val[0] = 1 @@ -297,30 +348,3 @@ func makeValue(s *state, c byte) error { s.valReady = true return nil } - -func SetOpener(driver string, loc string, cleanup bool) { - donefunc := func() error { return nil } - if cleanup { - donefunc = func() error { return os.RemoveAll(loc) } - } - if driver == "badger" { - DsOpener = func() (ds.TxnDatastore, Donefunc) { - d, err := badger.NewDatastore(loc, &badger.DefaultOptions) - if err != nil { - panic("could not create db instance") - } - return d, donefunc - } - } else if driver == "level" { - DsOpener = func() (ds.TxnDatastore, Donefunc) { - d, err := leveldb.NewDatastore(loc, &leveldb.Options{}) - if err != nil { - panic("could not create db instance") - } - return d, donefunc - } - } else { - // TODO - panic("unknown database") - } -} From d2ec4b247c4539ec6d5400b1f994a4c7758a143c Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 6 Apr 2020 11:18:54 -0700 Subject: [PATCH 09/12] linting --- fuzz/cmd/compare/main.go | 2 -- fuzz/cmd/generate/main.go | 10 +++++++--- fuzz/fuzzer.go | 20 ++++++++++---------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/fuzz/cmd/compare/main.go b/fuzz/cmd/compare/main.go index be78146..77a2d8c 100644 --- a/fuzz/cmd/compare/main.go +++ b/fuzz/cmd/compare/main.go @@ -101,6 +101,4 @@ func main() { } fmt.Printf("Done\n") - - return } diff --git a/fuzz/cmd/generate/main.go b/fuzz/cmd/generate/main.go index 84d64c3..f316d42 100644 --- a/fuzz/cmd/generate/main.go +++ b/fuzz/cmd/generate/main.go @@ -1,4 +1,4 @@ -// This file is invoked at the fuzzer level by `go generate` +// This file is invoked by `go generate` package main import ( @@ -36,17 +36,21 @@ func main() { name := nameComponents[len(nameComponents)-1] f, err := os.Create(fmt.Sprintf("provider_%s.go", name)) if err != nil { - fmt.Fprintf(os.Stderr, "failed to create provider file: %v", err) + fmt.Fprintf(os.Stderr, "failed to create provider file: %v\n", err) os.Exit(1) } defer f.Close() - provideTemplate.Execute(f, struct { + err = provideTemplate.Execute(f, struct { Package string PackageName string }{ Package: provider, PackageName: name, }) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to write provider: %v\n", err) + os.Exit(1) + } } } diff --git a/fuzz/fuzzer.go b/fuzz/fuzzer.go index b574a8f..657a7f4 100644 --- a/fuzz/fuzzer.go +++ b/fuzz/fuzzer.go @@ -46,7 +46,7 @@ type RunState struct { keyCache [128]ds.Key cachedKeys int32 - ctr int32 + ctr int32 //nolint:structcheck,unused } // DB returns the datastore being driven by this instance @@ -192,7 +192,7 @@ const ( opMax ) -func threadDriver(ctx context.Context, runState *RunState, cmnds chan byte) error { +func threadDriver(ctx context.Context, runState *RunState, cmnds chan byte) { defer runState.wg.Done() s := threadState{} s.RunState = runState @@ -203,11 +203,11 @@ func threadDriver(ctx context.Context, runState *RunState, cmnds chan byte) erro select { case c, ok := <-cmnds: if !ok { - return nil + return } _ = nextState(&s, c) case <-ctx.Done(): - return nil + return } } } @@ -220,21 +220,21 @@ func nextState(s *threadState, c byte) error { if !s.keyReady { return makeKey(s, c) } - s.reader.Get(s.key) + _, _ = s.reader.Get(s.key) reset(s) return nil } else if s.op == opHas { if !s.keyReady { return makeKey(s, c) } - s.reader.Has(s.key) + _, _ = s.reader.Has(s.key) reset(s) return nil } else if s.op == opGetSize { if !s.keyReady { return makeKey(s, c) } - s.reader.GetSize(s.key) + _, _ = s.reader.GetSize(s.key) reset(s) return nil } else if s.op == opQuery { @@ -255,14 +255,14 @@ func nextState(s *threadState, c byte) error { if !s.valReady { return makeValue(s, c) } - s.writer.Put(s.key, s.val) + _ = s.writer.Put(s.key, s.val) reset(s) return nil } else if s.op == opDelete { if !s.keyReady { return makeKey(s, c) } - s.writer.Delete(s.key) + _ = s.writer.Delete(s.key) reset(s) return nil } else if s.op == opNewTX { @@ -297,7 +297,7 @@ func nextState(s *threadState, c byte) error { if !s.keyReady { return makeKey(s, c) } - s.RunState.inst.Sync(s.key) + _ = s.RunState.inst.Sync(s.key) reset(s) return nil } From d3093d8d133ab22e6dfd5d00baeba36a9d975476 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 24 Apr 2020 09:18:57 -0700 Subject: [PATCH 10/12] submodule --- fuzz/README.md | 5 ++ fuzz/cmd/generate/main.go | 3 ++ fuzz/fuzzer.go | 2 +- fuzz/go.mod | 9 ++++ fuzz/go.sum | 93 +++++++++++++++++++++++++++++++++ go.mod | 7 +-- go.sum | 105 ++------------------------------------ 7 files changed, 117 insertions(+), 107 deletions(-) create mode 100644 fuzz/go.mod create mode 100644 fuzz/go.sum diff --git a/fuzz/README.md b/fuzz/README.md index 39ac629..c3c717e 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -27,3 +27,8 @@ Finally, build the fuzzing artifact, and fuzz go-fuzz-build go-fuzz ``` + +If you don't have `go-fuzz` installed, execute +``` +go get -u github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build +``` diff --git a/fuzz/cmd/generate/main.go b/fuzz/cmd/generate/main.go index f316d42..011b911 100644 --- a/fuzz/cmd/generate/main.go +++ b/fuzz/cmd/generate/main.go @@ -25,6 +25,9 @@ func main() { for _, provider := range providers { provider = strings.TrimSpace(provider) + if len(provider) == 0 { + continue + } cmd := exec.Command("go", "get", provider) err := cmd.Run() if err != nil { diff --git a/fuzz/fuzzer.go b/fuzz/fuzzer.go index 657a7f4..8252563 100644 --- a/fuzz/fuzzer.go +++ b/fuzz/fuzzer.go @@ -112,7 +112,7 @@ func Open(driver string, location string, cleanup bool) (*RunState, error) { // Fuzz is a go-fuzzer compatible input point for replaying // data (interpreted as a script of commands) -// to kown ipfs datastore implementations +// to known ipfs datastore implementations func Fuzz(data []byte) int { var impls []string for impl := range openers { diff --git a/fuzz/go.mod b/fuzz/go.mod new file mode 100644 index 0000000..7305fda --- /dev/null +++ b/fuzz/go.mod @@ -0,0 +1,9 @@ +module github.com/ipfs/go-datastore/fuzz + +go 1.14 + +require ( + github.com/ipfs/go-datastore v0.4.4 + github.com/ipfs/go-ds-badger v0.2.4 + github.com/spf13/pflag v1.0.3 +) diff --git a/fuzz/go.sum b/fuzz/go.sum new file mode 100644 index 0000000..93c1b74 --- /dev/null +++ b/fuzz/go.sum @@ -0,0 +1,93 @@ +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.6.1 h1:w9pSFNSdq/JPM1N12Fz/F/bzo993Is1W+Q7HjPzi7yg= +github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= +github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8X8= +github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-ds-badger v0.2.4 h1:UPGB0y7luFHk+mY/tUZrif/272M8o+hFsW+avLUeWrM= +github.com/ipfs/go-ds-badger v0.2.4/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= +github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= +github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= +github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= +github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= +github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/go.mod b/go.mod index dc4b59d..ba525a9 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,10 @@ module github.com/ipfs/go-datastore require ( - github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813 // indirect github.com/google/uuid v1.1.1 - github.com/ipfs/go-ds-badger v0.2.1 - github.com/ipfs/go-ds-leveldb v0.4.2 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8 - github.com/jbenet/goprocess v0.1.3 - github.com/spf13/pflag v1.0.3 + github.com/jbenet/goprocess v0.1.4 + github.com/kr/pretty v0.2.0 // indirect golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 ) diff --git a/go.sum b/go.sum index 408ce6b..1dd247b 100644 --- a/go.sum +++ b/go.sum @@ -1,111 +1,14 @@ -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo= -github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813 h1:NgO45/5mBLRVfiXerEFzH6ikcZ7DNRPS639xFg3ENzU= -github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= -github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= -github.com/ipfs/go-ds-badger v0.2.1 h1:RsC9DDlwFhFdfT+s2PeC8joxbSp2YMufK8w/RBOxKtk= -github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= -github.com/ipfs/go-ds-leveldb v0.4.2 h1:QmQoAJ9WkPMUfBLnu1sBVy0xWWlJPg0m4kRAiJL9iaw= -github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= -github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8 h1:NAviDvJ0WXgD+yiL2Rj35AmnfgI11+pHXbdciD917U0= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= -github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= -github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= -github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8 h1:bspPhN+oKYFk5fcGNuQzp6IGzYQSenLEgH3s6jkXrWw= -github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= -github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= -github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= +github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= -github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= -github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 0442b81e9252cdb51b13423f4228bf13d8ef302e Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 24 Apr 2020 09:47:24 -0700 Subject: [PATCH 11/12] add replace --- fuzz/go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fuzz/go.mod b/fuzz/go.mod index 7305fda..a080762 100644 --- a/fuzz/go.mod +++ b/fuzz/go.mod @@ -2,6 +2,8 @@ module github.com/ipfs/go-datastore/fuzz go 1.14 +replace github.com/ipfs/go-datastore => ../ + require ( github.com/ipfs/go-datastore v0.4.4 github.com/ipfs/go-ds-badger v0.2.4 From ed90f4ae26936efad6c0b91ffe41e4437ac71dfc Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 24 Apr 2020 10:56:40 -0700 Subject: [PATCH 12/12] clean readme --- fuzz/README.md | 12 +++--------- fuzz/go.sum | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/fuzz/README.md b/fuzz/README.md index c3c717e..715b026 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -8,13 +8,7 @@ implementations. Usage ---- -First, get the code -```golang -go get github.com/ipfs/go-datastore -cd go-datastore/fuzz -``` - -Next, configure the datastores to fuzz (from this directory) +First, configure the datastores to fuzz (from this directory): ```golang // either run via `go run` go run ./cmd/generate github.com/ipfs/go-ds-badger @@ -22,13 +16,13 @@ go run ./cmd/generate github.com/ipfs/go-ds-badger DS_PROVIDERS="github.com/ipfs/go-ds-badger" go generate ``` -Finally, build the fuzzing artifact, and fuzz +Then, build the fuzzing artifact and fuzz: ```golang go-fuzz-build go-fuzz ``` -If you don't have `go-fuzz` installed, execute +If you don't have `go-fuzz` installed, it can be acquired as: ``` go get -u github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build ``` diff --git a/fuzz/go.sum b/fuzz/go.sum index 93c1b74..6be1ad6 100644 --- a/fuzz/go.sum +++ b/fuzz/go.sum @@ -23,6 +23,7 @@ github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -57,6 +58,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -75,19 +77,36 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=