diff --git a/CHANGELOG.md b/CHANGELOG.md index 71271a1..0e0598b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The following emojis are used to highlight certain changes: ### Added - Ability to specify the maximum blocksize that bitswap will replace WantHave with WantBlock responses, and to disable replacement when set to zero. [#165](https://github.com/ipfs/rainbow/pull/165) +- Support use and configuration of pebble as [datastore](https://github.com/ipfs/rainbow/blob/main/docs/blockstores.md). Pebble provides a high-performance alternative to badger. Options are available to configure key tuning parameters (`pebble-*` in `rainbow --help`). ### Changed diff --git a/docs/blockstores.md b/docs/blockstores.md index 577d28e..0761bbf 100644 --- a/docs/blockstores.md +++ b/docs/blockstores.md @@ -20,6 +20,16 @@ filesystem (i.e. not just how bytes are stored on disk but file and directory st optimizations to be had in selection of the filesystem and disk types. For example, choosing a filesystem that enables putting file metadata on a fast SSD while keeping the actual data on a slower disk might ease various lookup types. +## Pebble + +`rainbow` ships with [Pebble](https://github.com/cockroachdb/pebble) (version in `go.mod`) + +The main reasons to choose Pebble compared to FlatFS are: +- Much faster with reasonable configuration +- It comes with the ability to compress data on disk +- Native bloom filters +- Highly configurable to tune performance to your needs + ## Badger `rainbow` ships with [Badger-v4](https://github.com/dgraph-io/badger). @@ -30,4 +40,4 @@ The main reasons to choose Badger compared to FlatFS are: - Native bloom filters The main difficulty with Badger is that its internal garbage collection functionality (not `rainbow`'s) is dependent on -workload which makes it difficult to ahead-of-time judge the kinds of capacity you need. \ No newline at end of file +workload which makes it difficult to ahead-of-time judge the kinds of capacity you need. diff --git a/go.mod b/go.mod index 3a5560a..c01e7d6 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/ipfs/rainbow go 1.22 require ( + github.com/cockroachdb/pebble v1.1.2 github.com/coreos/go-systemd/v22 v22.5.0 github.com/dgraph-io/badger/v4 v4.3.0 github.com/dustin/go-humanize v1.0.1 @@ -16,6 +17,7 @@ require ( github.com/ipfs/go-ds-badger4 v0.1.5 github.com/ipfs/go-ds-flatfs v0.5.1 github.com/ipfs/go-ds-leveldb v0.5.0 + github.com/ipfs/go-ds-pebble v0.3.2-0.20241002075519-c174835dc84a github.com/ipfs/go-ipfs-delay v0.0.1 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-metrics-interface v0.0.1 @@ -49,6 +51,7 @@ require ( ) require ( + github.com/DataDog/zstd v1.4.5 // indirect github.com/Jorropo/jsync v1.0.1 // indirect github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect @@ -56,6 +59,11 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cockroachdb/errors v1.11.3 // indirect + github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect @@ -70,6 +78,7 @@ require ( github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.4 // indirect + github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect @@ -116,6 +125,8 @@ require ( github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/koron/go-ssdp v0.0.4 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-doh-resolver v0.4.0 // indirect @@ -175,6 +186,7 @@ require ( github.com/quic-go/quic-go v0.45.2 // indirect github.com/quic-go/webtransport-go v0.8.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/samber/lo v1.39.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/go.sum b/go.sum index 6a80d14..cd05d7c 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU= github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -46,6 +48,20 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= +github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= +github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= @@ -64,6 +80,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lV github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 h1:ZFUue+PNxmHlu7pYv+IYMtqlaO/0VwaGEqKepZf9JpA= github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -114,10 +131,14 @@ github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyT github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I= github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s= github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -263,6 +284,8 @@ github.com/ipfs/go-ds-flatfs v0.5.1/go.mod h1:RWTV7oZD/yZYBKdbVIFXTX2fdY2Tbvl94N github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUNumo= github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= +github.com/ipfs/go-ds-pebble v0.3.2-0.20241002075519-c174835dc84a h1:OouXE/TSvgFP3Scv/3WA1brxKC+FM40EeusWnLGilgY= +github.com/ipfs/go-ds-pebble v0.3.2-0.20241002075519-c174835dc84a/go.mod h1:ZyYU+weIni+4NG/Yjva+cPkU3ghlsU1HA2R/VLHJ9sM= github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ= github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= @@ -520,6 +543,8 @@ github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9K github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pion/datachannel v1.5.8 h1:ph1P1NsGkazkjrvyMfhRBUAWMxugJjq2HfQifaOoSNo= github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu8QzbL3tI= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= @@ -562,6 +587,7 @@ github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I= github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -596,6 +622,7 @@ github.com/quic-go/webtransport-go v0.8.0/go.mod h1:N99tjprW432Ut5ONql/aUhSLT0YV github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 h1:18kd+8ZUlt/ARXhljq+14TwAoKa61q6dX8jtwOf6DH8= diff --git a/main.go b/main.go index f3d1f29..816a3c1 100644 --- a/main.go +++ b/main.go @@ -184,7 +184,7 @@ Generate an identity seed and launch a gateway: Name: "inmem-block-cache", Value: 1 << 30, EnvVars: []string{"RAINBOW_INMEM_BLOCK_CACHE"}, - Usage: "Size of the in-memory block cache (currently only used for badger). 0 to disable (disables compression on disk too)", + Usage: "Size of the in-memory block cache (currently only used for pebble and badger). 0 to disable (disables compression on disk too)", }, &cli.Uint64Flag{ Name: "libp2p-max-memory", @@ -246,7 +246,7 @@ Generate an identity seed and launch a gateway: Name: "blockstore", Value: "flatfs", EnvVars: []string{"RAINBOW_BLOCKSTORE"}, - Usage: "Type of blockstore to use, such as flatfs or badger. See https://github.com/ipfs/rainbow/blob/main/docs/blockstores.md for more details", + Usage: "Type of blockstore to use, such as flatfs or pebble. See https://github.com/ipfs/rainbow/blob/main/docs/blockstores.md for more details", }, &cli.DurationFlag{ Name: "ipns-max-cache-ttl", @@ -315,6 +315,66 @@ Generate an identity seed and launch a gateway: EnvVars: []string{"RAINBOW_SAMPLING_FRACTION"}, Usage: "Rate at which to sample gateway requests. Does not include traceheaders which will always sample", }, + &cli.IntFlag{ + Name: "pebble-bytes-per-sync", + Value: 0, + EnvVars: []string{"PEBBLE_BYTES_PER_SYNC"}, + Usage: "Sync sstables periodically in order to smooth out writes to disk", + }, + &cli.BoolFlag{ + Name: "pebble-disable-wal", + Value: false, + EnvVars: []string{"PEBBLE_DISABLE_WAL"}, + Usage: "Disable the write-ahead log (WAL) at expense of prohibiting crash recoveryfg", + }, + &cli.IntFlag{ + Name: "pebble-l0-compaction-threshold", + Value: 0, + EnvVars: []string{"PEBBLE_L0_COMPACTION_THRESHOLD"}, + Usage: "Count of L0 files necessary to trigger an L0 compaction", + }, + &cli.IntFlag{ + Name: "pebble-l0-stop-writes-threshold", + Value: 0, + EnvVars: []string{"PEBBLE_L0_STOP_WRITES_THRESHOLD"}, + Usage: "Limit on L0 read-amplification, computed as the number of L0 sublevels", + }, + &cli.Int64Flag{ + Name: "pebble-lbase-max-bytes", + Value: 0, + EnvVars: []string{"PEBBLE_LBASE_MAX_BYTES"}, + Usage: "Maximum number of bytes for LBase. The base level is the level which L0 is compacted into", + }, + &cli.Uint64Flag{ + Name: "pebble-mem-table-size", + Value: 0, + EnvVars: []string{"PEBBLE_MEM_TABLE_SIZE"}, + Usage: "Size of a MemTable in steady state. The actual MemTable size starts at min(256KB, MemTableSize) and doubles for each subsequent MemTable up to MemTableSize", + }, + &cli.IntFlag{ + Name: "pebble-mem-table-stop-writes-threshold", + Value: 0, + EnvVars: []string{"PEBBLE_MEM_TABLE_STOP_WRITES_THRESHOLD"}, + Usage: "Limit on the number of queued of MemTables", + }, + &cli.IntFlag{ + Name: "pebble-wal-bytes-per-sync", + Value: 0, + EnvVars: []string{"PEBBLE_WAL_BYTES_PER_SYNC"}, + Usage: "Sets the number of bytes to write to a WAL before calling Sync on it in the background", + }, + &cli.IntFlag{ + Name: "pebble-max-concurrent-compactions", + Value: 0, + EnvVars: []string{"PEBBLE_MAX_CONCURRENT_COMPACTIONS"}, + Usage: "Maximum number of concurrent compactions", + }, + &cli.DurationFlag{ + Name: "pebble-wal-min-sync-interval", + Value: 0, + EnvVars: []string{"PEBBLE_WAL_MIN_SYNC_INTERVAL"}, + Usage: "Sets the minimum duration between syncs of the WAL", + }, } app.Commands = []*cli.Command{ @@ -461,8 +521,19 @@ share the same seed as long as the indexes are different. GCThreshold: cctx.Float64("gc-threshold"), ListenAddrs: cctx.StringSlice("libp2p-listen-addrs"), TracingAuthToken: cctx.String("tracing-auth"), - } + // Pebble config + BytesPerSync: cctx.Int("pebble-bytes-per-sync"), + DisableWAL: cctx.Bool("pebble-disable-wal"), + L0CompactionThreshold: cctx.Int("pebble-l0-compaction-threshold"), + L0StopWritesThreshold: cctx.Int("pebble-l0-stop-writes-threshold"), + LBaseMaxBytes: cctx.Int64("pebble-lbase-max-bytes"), + MemTableSize: cctx.Uint64("pebble-mem-table-size"), + MemTableStopWritesThreshold: cctx.Int("pebble-mem-table-stop-writes-threshold"), + WALBytesPerSync: cctx.Int("pebble-wal-Bytes-per-sync"), + MaxConcurrentCompactions: cctx.Int("pebble-max-concurrent-compactions"), + WALMinSyncInterval: time.Second * time.Duration(cctx.Int("pebble-wal-min-sync-interval-sec")), + } var gnd *Node goLog.Infof("Rainbow config: %+v", cfg) diff --git a/setup.go b/setup.go index fd18785..877d978 100644 --- a/setup.go +++ b/setup.go @@ -10,6 +10,7 @@ import ( "path/filepath" "time" + "github.com/cockroachdb/pebble" "github.com/dgraph-io/badger/v4" "github.com/dgraph-io/badger/v4/options" nopfs "github.com/ipfs-shipyard/nopfs" @@ -27,6 +28,7 @@ import ( "github.com/ipfs/go-datastore" badger4 "github.com/ipfs/go-ds-badger4" flatfs "github.com/ipfs/go-ds-flatfs" + pebbleds "github.com/ipfs/go-ds-pebble" mprome "github.com/ipfs/go-metrics-prometheus" "github.com/ipfs/go-unixfsnode" dagpb "github.com/ipld/go-codec-dagpb" @@ -133,6 +135,18 @@ type Config struct { TracingAuthToken string disableMetrics bool // only meant to be used during testing + + // Pebble config values + BytesPerSync int + DisableWAL bool + L0CompactionThreshold int + L0StopWritesThreshold int + LBaseMaxBytes int64 + MemTableSize uint64 + MemTableStopWritesThreshold int + WALBytesPerSync int + MaxConcurrentCompactions int + WALMinSyncInterval time.Duration } func SetupNoLibp2p(ctx context.Context, cfg Config, dnsCache *cachedDNS) (*Node, error) { @@ -350,6 +364,11 @@ func setupDatastore(cfg Config) (datastore.Batching, error) { switch cfg.BlockstoreType { case "flatfs": return flatfs.CreateOrOpen(filepath.Join(cfg.DataDir, "flatfs"), flatfs.NextToLast(3), false) + case "pebble": + return pebbleds.NewDatastore(filepath.Join(cfg.DataDir, "pebbleds"), + pebbleds.WithCacheSize(cfg.InMemBlockCache), + pebbleds.WithPebbleOpts(getPebbleOpts(cfg)), + ) case "badger": badgerOpts := badger.DefaultOptions("") badgerOpts.CompactL0OnClose = false @@ -557,3 +576,24 @@ func (s *switchingBlockstore) HashOnRead(enabled bool) { } var _ blockstore.Blockstore = (*switchingBlockstore)(nil) + +func getPebbleOpts(cfg Config) *pebble.Options { + opts := &pebble.Options{ + BytesPerSync: cfg.BytesPerSync, + DisableWAL: cfg.DisableWAL, + L0CompactionThreshold: cfg.L0CompactionThreshold, + L0StopWritesThreshold: cfg.L0StopWritesThreshold, + LBaseMaxBytes: cfg.LBaseMaxBytes, + MemTableSize: cfg.MemTableSize, + MemTableStopWritesThreshold: cfg.MemTableStopWritesThreshold, + WALBytesPerSync: cfg.WALBytesPerSync, + } + if cfg.MaxConcurrentCompactions != 0 { + opts.MaxConcurrentCompactions = func() int { return cfg.MaxConcurrentCompactions } + } + if cfg.WALMinSyncInterval != 0 { + opts.WALMinSyncInterval = func() time.Duration { return cfg.WALMinSyncInterval } + } + + return opts +}