Merge pull request #7 from bnb-chain/develop

release: draft new release v0.8.1-alpha.1

backend_test.go

@@ -13,7 +13,7 @@ import (
 // Register a test backend for PrefixDB as well, with some unrelated junk data
 func init() {
 	//nolint: errcheck
-	registerDBCreator("prefixdb", func(name, dir string) (DB, error) {
+	registerDBCreator("prefixdb", func(name, dir string, opts ...*NewDatabaseOption) (DB, error) {
 		mdb := NewMemDB()
 		mdb.Set([]byte("a"), []byte{1})
 		mdb.Set([]byte("b"), []byte{2})

badger_db.go

@@ -14,7 +14,7 @@ import (
 
 func init() { registerDBCreator(BadgerDBBackend, badgerDBCreator, true) }
 
-func badgerDBCreator(dbName, dir string) (DB, error) {
+func badgerDBCreator(dbName, dir string, opts ...*NewDatabaseOption) (DB, error) {
 	return NewBadgerDB(dbName, dir)
 }
 

boltdb.go

@@ -17,7 +17,7 @@ var (
 )
 
 func init() {
-	registerDBCreator(BoltDBBackend, func(name, dir string) (DB, error) {
+	registerDBCreator(BoltDBBackend, func(name, dir string, opts ...*NewDatabaseOption) (DB, error) {
 		return NewBoltDB(name, dir)
 	}, false)
 }

cleveldb.go

@@ -11,7 +11,7 @@ import (
 )
 
 func init() {
-	dbCreator := func(name string, dir string) (DB, error) {
+	dbCreator := func(name string, dir string, opts ...*NewDatabaseOption) (DB, error) {
 		return NewCLevelDB(name, dir)
 	}
 	registerDBCreator(CLevelDBBackend, dbCreator, false)

db.go

@@ -37,7 +37,7 @@ const (
 	BadgerDBBackend BackendType = "badgerdb"
 )
 
-type dbCreator func(name string, dir string) (DB, error)
+type dbCreator func(name string, dir string, opts ...*NewDatabaseOption) (DB, error)
 
 var backends = map[BackendType]dbCreator{}
 
@@ -50,7 +50,7 @@ func registerDBCreator(backend BackendType, creator dbCreator, force bool) {
 }
 
 // NewDB creates a new database of type backend with the given name.
-func NewDB(name string, backend BackendType, dir string) (DB, error) {
+func NewDB(name string, backend BackendType, dir string, opts ...*NewDatabaseOption) (DB, error) {
 	dbCreator, ok := backends[backend]
 	if !ok {
 		keys := make([]string, 0, len(backends))
@@ -61,7 +61,7 @@ func NewDB(name string, backend BackendType, dir string) (DB, error) {
 			backend, strings.Join(keys, ","))
 	}
 
-	db, err := dbCreator(name, dir)
+	db, err := dbCreator(name, dir, opts...)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize database: %w", err)
 	}

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.3 // indirect
 	github.com/klauspost/compress v1.12.3 // indirect
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/pkg/errors v0.8.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/net v0.9.0 // indirect

go.sum

@@ -73,6 +73,8 @@ github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9k
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

goleveldb.go

@@ -6,13 +6,20 @@ import (
 
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/errors"
+	"github.com/syndtr/goleveldb/leveldb/filter"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
+const (
+	// minCache is the minimum amount of memory in megabytes to allocate to leveldb
+	// read and write caching, split half and half.
+	minCache = 16
+)
+
 func init() {
-	dbCreator := func(name string, dir string) (DB, error) {
-		return NewGoLevelDB(name, dir)
+	dbCreator := func(name string, dir string, opts ...*NewDatabaseOption) (DB, error) {
+		return NewGoLevelDB(name, dir, opts...)
 	}
 	registerDBCreator(GoLevelDBBackend, dbCreator, false)
 }
@@ -23,8 +30,31 @@ type GoLevelDB struct {
 
 var _ DB = (*GoLevelDB)(nil)
 
-func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) {
-	return NewGoLevelDBWithOpts(name, dir, nil)
+func NewGoLevelDB(name string, dir string, opts ...*NewDatabaseOption) (*GoLevelDB, error) {
+	externalOpt := &NewDatabaseOption{}
+	// TODO: use option pattern
+	if len(opts) > 0 {
+		externalOpt = opts[0]
+	}
+	cache := externalOpt.Cache / opt.MiB
+	if cache < minCache {
+		cache = minCache
+	}
+	handles := 200
+	if externalOpt.Handles > handles {
+		handles = externalOpt.Handles
+	}
+	filterSize := 10
+	if externalOpt.Filter > filterSize {
+		filterSize = externalOpt.Filter
+	}
+
+	return NewGoLevelDBWithOpts(name, dir, &opt.Options{
+		OpenFilesCacheCapacity: handles,
+		BlockCacheCapacity:     cache / 2 * opt.MiB,
+		WriteBuffer:            cache / 4 * opt.MiB, // Two of these are used internally
+		Filter:                 filter.NewBloomFilter(filterSize),
+	})
 }
 
 func NewGoLevelDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB, error) {

memdb.go

@@ -14,7 +14,7 @@ const (
 )
 
 func init() {
-	registerDBCreator(MemDBBackend, func(name, dir string) (DB, error) {
+	registerDBCreator(MemDBBackend, func(name, dir string, opts ...*NewDatabaseOption) (DB, error) {
 		return NewMemDB(), nil
 	}, false)
 }

option.go

@@ -0,0 +1,7 @@
+package db
+
+type NewDatabaseOption struct {
+	Cache   int
+	Handles int
+	Filter  int
+}

rocksdb.go

@@ -12,7 +12,7 @@ import (
 )
 
 func init() {
-	dbCreator := func(name string, dir string) (DB, error) {
+	dbCreator := func(name string, dir string, opts ...*NewDatabaseOption) (DB, error) {
 		return NewRocksDB(name, dir)
 	}
 	registerDBCreator(RocksDBBackend, dbCreator, false)