From b0e93377dd4da7da28842746f35ed8baa57e67c1 Mon Sep 17 00:00:00 2001 From: Roshan <48975233+Pythonberg1997@users.noreply.github.com> Date: Thu, 6 Jul 2023 17:22:49 +0800 Subject: [PATCH] perf: optimize the goleveldb option (#3) --- backend_test.go | 2 +- badger_db.go | 2 +- boltdb.go | 2 +- cleveldb.go | 2 +- db.go | 6 +++--- go.mod | 1 + go.sum | 2 ++ goleveldb.go | 38 ++++++++++++++++++++++++++++++++++---- memdb.go | 2 +- option.go | 7 +++++++ rocksdb.go | 2 +- 11 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 option.go diff --git a/backend_test.go b/backend_test.go index 0f19241..338ee58 100644 --- a/backend_test.go +++ b/backend_test.go @@ -13,7 +13,7 @@ import ( // Register a test backend for PrefixDB as well, with some unrelated junk data func init() { //nolint: errcheck - registerDBCreator("prefixdb", func(name, dir string) (DB, error) { + registerDBCreator("prefixdb", func(name, dir string, opts ...*NewDatabaseOption) (DB, error) { mdb := NewMemDB() mdb.Set([]byte("a"), []byte{1}) mdb.Set([]byte("b"), []byte{2}) diff --git a/badger_db.go b/badger_db.go index ecdfca1..6d9434f 100644 --- a/badger_db.go +++ b/badger_db.go @@ -14,7 +14,7 @@ import ( func init() { registerDBCreator(BadgerDBBackend, badgerDBCreator, true) } -func badgerDBCreator(dbName, dir string) (DB, error) { +func badgerDBCreator(dbName, dir string, opts ...*NewDatabaseOption) (DB, error) { return NewBadgerDB(dbName, dir) } diff --git a/boltdb.go b/boltdb.go index ffebbca..bed47b0 100644 --- a/boltdb.go +++ b/boltdb.go @@ -17,7 +17,7 @@ var ( ) func init() { - registerDBCreator(BoltDBBackend, func(name, dir string) (DB, error) { + registerDBCreator(BoltDBBackend, func(name, dir string, opts ...*NewDatabaseOption) (DB, error) { return NewBoltDB(name, dir) }, false) } diff --git a/cleveldb.go b/cleveldb.go index 7896730..f73370d 100644 --- a/cleveldb.go +++ b/cleveldb.go @@ -11,7 +11,7 @@ import ( ) func init() { - dbCreator := func(name string, dir string) (DB, error) { + dbCreator := func(name string, dir string, opts ...*NewDatabaseOption) (DB, error) { return NewCLevelDB(name, dir) } registerDBCreator(CLevelDBBackend, dbCreator, false) diff --git a/db.go b/db.go index 4d518c0..dab9c9a 100644 --- a/db.go +++ b/db.go @@ -37,7 +37,7 @@ const ( BadgerDBBackend BackendType = "badgerdb" ) -type dbCreator func(name string, dir string) (DB, error) +type dbCreator func(name string, dir string, opts ...*NewDatabaseOption) (DB, error) var backends = map[BackendType]dbCreator{} @@ -50,7 +50,7 @@ func registerDBCreator(backend BackendType, creator dbCreator, force bool) { } // NewDB creates a new database of type backend with the given name. -func NewDB(name string, backend BackendType, dir string) (DB, error) { +func NewDB(name string, backend BackendType, dir string, opts ...*NewDatabaseOption) (DB, error) { dbCreator, ok := backends[backend] if !ok { keys := make([]string, 0, len(backends)) @@ -61,7 +61,7 @@ func NewDB(name string, backend BackendType, dir string) (DB, error) { backend, strings.Join(keys, ",")) } - db, err := dbCreator(name, dir) + db, err := dbCreator(name, dir, opts...) if err != nil { return nil, fmt.Errorf("failed to initialize database: %w", err) } diff --git a/go.mod b/go.mod index d05d01f..6be9b4d 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.3 // indirect github.com/klauspost/compress v1.12.3 // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pkg/errors v0.8.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/net v0.9.0 // indirect diff --git a/go.sum b/go.sum index e7a577d..acca688 100644 --- a/go.sum +++ b/go.sum @@ -73,6 +73,8 @@ github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9k github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/goleveldb.go b/goleveldb.go index fd1bffd..f73fe0f 100644 --- a/goleveldb.go +++ b/goleveldb.go @@ -6,13 +6,20 @@ import ( "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/syndtr/goleveldb/leveldb/filter" "github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/util" ) +const ( + // minCache is the minimum amount of memory in megabytes to allocate to leveldb + // read and write caching, split half and half. + minCache = 16 +) + func init() { - dbCreator := func(name string, dir string) (DB, error) { - return NewGoLevelDB(name, dir) + dbCreator := func(name string, dir string, opts ...*NewDatabaseOption) (DB, error) { + return NewGoLevelDB(name, dir, opts...) } registerDBCreator(GoLevelDBBackend, dbCreator, false) } @@ -23,8 +30,31 @@ type GoLevelDB struct { var _ DB = (*GoLevelDB)(nil) -func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) { - return NewGoLevelDBWithOpts(name, dir, nil) +func NewGoLevelDB(name string, dir string, opts ...*NewDatabaseOption) (*GoLevelDB, error) { + externalOpt := &NewDatabaseOption{} + // TODO: use option pattern + if len(opts) > 0 { + externalOpt = opts[0] + } + cache := externalOpt.Cache / opt.MiB + if cache < minCache { + cache = minCache + } + handles := 200 + if externalOpt.Handles > handles { + handles = externalOpt.Handles + } + filterSize := 10 + if externalOpt.Filter > filterSize { + filterSize = externalOpt.Filter + } + + return NewGoLevelDBWithOpts(name, dir, &opt.Options{ + OpenFilesCacheCapacity: handles, + BlockCacheCapacity: cache / 2 * opt.MiB, + WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally + Filter: filter.NewBloomFilter(filterSize), + }) } func NewGoLevelDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB, error) { diff --git a/memdb.go b/memdb.go index 2bfa0d1..4d161ab 100644 --- a/memdb.go +++ b/memdb.go @@ -14,7 +14,7 @@ const ( ) func init() { - registerDBCreator(MemDBBackend, func(name, dir string) (DB, error) { + registerDBCreator(MemDBBackend, func(name, dir string, opts ...*NewDatabaseOption) (DB, error) { return NewMemDB(), nil }, false) } diff --git a/option.go b/option.go new file mode 100644 index 0000000..f32708a --- /dev/null +++ b/option.go @@ -0,0 +1,7 @@ +package db + +type NewDatabaseOption struct { + Cache int + Handles int + Filter int +} diff --git a/rocksdb.go b/rocksdb.go index 945491c..c6b56dc 100644 --- a/rocksdb.go +++ b/rocksdb.go @@ -12,7 +12,7 @@ import ( ) func init() { - dbCreator := func(name string, dir string) (DB, error) { + dbCreator := func(name string, dir string, opts ...*NewDatabaseOption) (DB, error) { return NewRocksDB(name, dir) } registerDBCreator(RocksDBBackend, dbCreator, false)