diff --git a/rocksdb.nim b/rocksdb.nim index af5609a..afd2d56 100644 --- a/rocksdb.nim +++ b/rocksdb.nim @@ -1,5 +1,5 @@ # Nim-RocksDB -# Copyright 2018 Status Research & Development GmbH +# Copyright 2018-2024 Status Research & Development GmbH # Licensed under either of # # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) @@ -35,7 +35,7 @@ type RocksDBInstance* = object db*: rocksdb_t backupEngine: rocksdb_backup_engine_t - options*: rocksdb_options_t + options*: seq[rocksdb_options_t] readOptions*: rocksdb_readoptions_t writeOptions: rocksdb_writeoptions_t dbPath: string # needed for clear() @@ -57,7 +57,7 @@ template validateColumnFamily( columnFamily: string): rocksdb_column_family_handle_t = if not db.columnFamilies.contains(columnFamily): - return err("rocksdb: invalid column family") + return err("rocksdb: unknown column family") let columnFamilyHandle= db.columnFamilies.getOrDefault(columnFamily) doAssert not columnFamilyHandle.isNil @@ -70,55 +70,58 @@ proc init*(rocks: var RocksDBInstance, cpus = countProcessors(), createIfMissing = true, maxOpenFiles = -1, - columnFamiliesNames: openArray[string] = @["default"]): RocksDBResult[void] = - rocks.options = rocksdb_options_create() + columnFamilyNames: openArray[string] = @["default"]): RocksDBResult[void] = + + for i in 0..columnFamilyNames.high: + rocks.options.add(rocksdb_options_create()) rocks.readOptions = rocksdb_readoptions_create() rocks.writeOptions = rocksdb_writeoptions_create() rocks.dbPath = dbPath - rocks.columnFamilyNames = columnFamiliesNames.allocCStringArray + rocks.columnFamilyNames = columnFamilyNames.allocCStringArray rocks.columnFamilies = newTable[cstring, rocksdb_column_family_handle_t]() - # Optimize RocksDB. This is the easiest way to get RocksDB to perform well: - rocksdb_options_increase_parallelism(rocks.options, cpus.int32) - # This requires snappy - disabled because rocksdb is not always compiled with - # snappy support (for example Fedora 28, certain Ubuntu versions) - # rocksdb_options_optimize_level_style_compaction(options, 0); - rocksdb_options_set_create_if_missing(rocks.options, uint8(createIfMissing)) - # default set to keep all files open (-1), allow setting it to a specific - # value, e.g. in case the application limit would be reached. - rocksdb_options_set_max_open_files(rocks.options, maxOpenFiles.cint) - # Enable creating column families if they do not exist - rocksdb_options_set_create_missing_column_families(rocks.options, uint8(true)) + for opts in rocks.options: + # Optimize RocksDB. This is the easiest way to get RocksDB to perform well: + rocksdb_options_increase_parallelism(opts, cpus.int32) + # This requires snappy - disabled because rocksdb is not always compiled with + # snappy support (for example Fedora 28, certain Ubuntu versions) + # rocksdb_options_optimize_level_style_compaction(options, 0); + rocksdb_options_set_create_if_missing(opts, uint8(createIfMissing)) + # default set to keep all files open (-1), allow setting it to a specific + # value, e.g. in case the application limit would be reached. + rocksdb_options_set_max_open_files(opts, maxOpenFiles.cint) + # Enable creating column families if they do not exist + rocksdb_options_set_create_missing_column_families(opts, uint8(true)) var - columnFamilyHandles = newSeq[rocksdb_column_family_handle_t](columnFamiliesNames.len) + columnFamilyHandles = newSeq[rocksdb_column_family_handle_t](columnFamilyNames.len) errors: cstring if readOnly: rocks.db = rocksdb_open_for_read_only_column_families( - rocks.options, + rocks.options[0], dbPath, - columnFamiliesNames.len().cint, + columnFamilyNames.len().cint, rocks.columnFamilyNames, - rocks.options.addr, # TODO: test this. Might need to turn this into array of options + rocks.options[0].addr, columnFamilyHandles[0].addr, 0'u8, errors.addr) else: rocks.db = rocksdb_open_column_families( - rocks.options, + rocks.options[0], dbPath, - columnFamiliesNames.len().cint, + columnFamilyNames.len().cint, rocks.columnFamilyNames, - rocks.options.addr, # TODO: test this. Might need to turn this into array of options + rocks.options[0].addr, columnFamilyHandles[0].addr, errors.addr) bailOnErrors() - for i in 0.. 0: unsafeAddr val[0] else: nil), csize_t(val.len), errors.addr) - bailOnErrors() + ok() proc contains*(db: RocksDBInstance, key: openArray[byte], columnFamily = "default"): RocksDBResult[bool] = @@ -252,7 +255,7 @@ proc del*( # This seems like a bad idea, but right now I don't want to # get sidetracked by this. --Adam - if not db.contains(key).get: + if not db.contains(key, columnFamily).get: return ok(false) var errors: cstring @@ -264,6 +267,7 @@ proc del*( csize_t(key.len), errors.addr) bailOnErrors() + ok(true) proc clear*(db: var RocksDBInstance): RocksDBResult[bool] = @@ -281,7 +285,7 @@ proc backup*(db: RocksDBInstance): RocksDBResult[void] = proc close*(db: var RocksDBInstance) = if not db.columnFamilies.isNil: - for k, v in db.columnFamilies: + for _, v in db.columnFamilies: rocksdb_column_family_handle_destroy(v) db.columnFamilies = nil @@ -289,14 +293,18 @@ proc close*(db: var RocksDBInstance) = db.columnFamilyNames.deallocCStringArray() db.columnFamilyNames = nil - template freeField(name) = - if db.`name`.isNil: - `rocksdb name destroy`(db.`name`) - db.`name` = nil + if not db.writeOptions.isNil: + rocksdb_writeoptions_destroy(db.writeOptions) + db.writeOptions = nil + + if not db.readOptions.isNil: + rocksdb_readoptions_destroy(db.readOptions) + db.readOptions = nil - freeField(writeOptions) - freeField(readOptions) - freeField(options) + if db.options.len() > 0: + for o in db.options: + rocksdb_options_destroy(o) + db.options = @[] if not db.backupEngine.isNil: rocksdb_backup_engine_close(db.backupEngine) diff --git a/rocksdb.nimble b/rocksdb.nimble index ee0e2d0..0eee7b9 100644 --- a/rocksdb.nimble +++ b/rocksdb.nimble @@ -9,7 +9,8 @@ mode = ScriptMode.Verbose ### Dependencies requires "nim >= 1.2.0", "stew", - "tempfile" + "tempfile", + "unittest2" proc test(args, path: string) = if not dirExists "build": diff --git a/tests/test_rocksdb.nim b/tests/test_rocksdb.nim index cc53458..0aee5b2 100644 --- a/tests/test_rocksdb.nim +++ b/tests/test_rocksdb.nim @@ -1,5 +1,5 @@ # Nim-RocksDB -# Copyright 2018-2019 Status Research & Development GmbH +# Copyright 2018-2024 Status Research & Development GmbH # Licensed under either of # # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) @@ -10,17 +10,16 @@ {.used.} import - os, unittest, + std/os, tempfile, + unittest2, ../rocksdb -type - MyDB = object - rocksdb: RocksDBInstance +proc initMyDb( + path: string, + readOnly = false, + columnFamilyNames = @["default"]): RocksDBInstance = -# TODO no tests for failures / error reporting - -proc initMyDb(path: string): MyDB = let dataDir = path / "data" backupsDir = path / "backups" @@ -28,50 +27,226 @@ proc initMyDb(path: string): MyDB = createDir(dataDir) createDir(backupsDir) - var s = result.rocksdb.init(dataDir, backupsDir) - doAssert s.isOk, $s + var s = result.init( + dataDir, + backupsDir, + readOnly = readOnly, + columnFamilyNames = columnFamilyNames) + doAssert s.isOk(), $s + suite "Nim API tests": - setup: + const + CF_DEFAULT = "default" + CF_OTHER = "other" + + let + key = @[byte(1), 2, 3, 4, 5] + otherKey = @[byte(1), 2, 3, 4, 5, 6] + val = @[byte(1), 2, 3, 4, 5] + + test "Basic operations": var dbDir = mkdtemp() db = initMyDb(dbDir) - teardown: - close(db.rocksdb) + var s = db.put(key, val) + check s.isOk() + + var bytes: seq[byte] + check db.get(key, proc(data: openArray[byte]) = bytes = @data)[] + check not db.get( + otherkey, proc(data: openArray[byte]) = bytes = @data)[] + + var r1 = db.getBytes(key) + check r1.isOk() and r1.value == val + + var r2 = db.getBytes(otherKey) + # there's no error string for missing keys + check r2.isOk() == false and r2.error.len == 0 + + var e1 = db.contains(key) + check e1.isOk() and e1.value == true + + var e2 = db.contains(otherKey) + check e2.isOk() and e2.value == false + + var d = db.del(key) + check d.isOk() and d.value == true + + e1 = db.contains(key) + check e1.isOk() and e1.value == false + + d = db.del(otherKey) + check d.isOk() and d.value == false + + close(db) + + # Open database in read only mode + block: + var + readOnlyDb = initMyDb(dbDir, readOnly = true) + + var r = readOnlyDb.contains(key) + check r.isOk() and r.value == false + + var r2 = readOnlyDb.put(key, @[123.byte]) + check r2.isErr() + + close(readOnlyDb) + removeDir(dbDir) - test "Basic operations": - proc test() = - let key = @[byte(1), 2, 3, 4, 5] - let otherKey = @[byte(1), 2, 3, 4, 5, 6] - let val = @[byte(1), 2, 3, 4, 5] + test "Basic operations - default column family": + var + dbDir = mkdtemp() + db = initMyDb(dbDir, columnFamilyNames = @[CF_DEFAULT]) + + var s = db.put(key, val, CF_DEFAULT) + check s.isOk() + + var bytes: seq[byte] + check db.get(key, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[] + check not db.get( + otherkey, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[] + + var r1 = db.getBytes(key) + check r1.isOk() and r1.value == val + + var r2 = db.getBytes(otherKey) + # there's no error string for missing keys + check r2.isOk() == false and r2.error.len == 0 + + var e1 = db.contains(key, CF_DEFAULT) + check e1.isOk() and e1.value == true + + var e2 = db.contains(otherKey, CF_DEFAULT) + check e2.isOk() and e2.value == false + + var d = db.del(key, CF_DEFAULT) + check d.isOk() and d.value == true + + e1 = db.contains(key, CF_DEFAULT) + check e1.isOk() and e1.value == false + + d = db.del(otherKey, CF_DEFAULT) + check d.isOk() and d.value == false + + close(db) - var s = db.rocksdb.put(key, val) - check s.isok + # Open database in read only mode + block: + var + readOnlyDb = initMyDb(dbDir, readOnly = true, columnFamilyNames = @[CF_DEFAULT]) - var bytes: seq[byte] - check db.rocksdb.get(key, proc(data: openArray[byte]) = bytes = @data)[] - check not db.rocksdb.get( - otherkey, proc(data: openArray[byte]) = bytes = @data)[] + var r = readOnlyDb.contains(key, CF_DEFAULT) + check r.isOk() and r.value == false - var r1 = db.rocksdb.getBytes(key) - check r1.isok and r1.value == val + var r2 = readOnlyDb.put(key, @[123.byte], CF_DEFAULT) + echo r2 + check r2.isErr() - var r2 = db.rocksdb.getBytes(otherKey) - # there's no error string for missing keys - check r2.isok == false and r2.error.len == 0 + close(readOnlyDb) + + removeDir(dbDir) + + test "Basic operations - multiple column families": + var + dbDir = mkdtemp() + db = initMyDb(dbDir, columnFamilyNames = @[CF_DEFAULT, CF_OTHER]) + + var s = db.put(key, val, CF_DEFAULT) + check s.isOk() + + var s2 = db.put(otherKey, val, CF_OTHER) + check s2.isOk() + + var bytes: seq[byte] + check db.get(key, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[] + check not db.get( + otherkey, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[] + + var bytes2: seq[byte] + check db.get(otherKey, proc(data: openArray[byte]) = bytes2 = @data, CF_OTHER)[] + check not db.get( + key, proc(data: openArray[byte]) = bytes2 = @data, CF_OTHER)[] + + var e1 = db.contains(key, CF_DEFAULT) + check e1.isOk() and e1.value == true + var e2 = db.contains(otherKey, CF_DEFAULT) + check e2.isOk() and e2.value == false + + var e3 = db.contains(key, CF_OTHER) + check e3.isOk() and e3.value == false + var e4 = db.contains(otherKey, CF_OTHER) + check e4.isOk() and e4.value == true + + var d = db.del(key, CF_DEFAULT) + check d.isOk() and d.value == true + e1 = db.contains(key, CF_DEFAULT) + check e1.isOk() and e1.value == false + d = db.del(otherKey, CF_DEFAULT) + check d.isOk() and d.value == false + + var d2 = db.del(key, CF_OTHER) + check d2.isOk() and d2.value == false + e3 = db.contains(key, CF_OTHER) + check e3.isOk() and e3.value == false + d2 = db.del(otherKey, CF_OTHER) + check d2.isOk() and d2.value == true + d2 = db.del(otherKey, CF_OTHER) + check d2.isOk() and d2.value == false + + db.close() + + # Open database in read only mode + block: + var + readOnlyDb = initMyDb(dbDir, readOnly = true, columnFamilyNames = @[CF_DEFAULT, CF_OTHER]) + + var r = readOnlyDb.contains(key, CF_OTHER) + check r.isOk() and r.value == false + + var r2 = readOnlyDb.put(key, @[123.byte], CF_OTHER) + check r2.isErr() + + close(readOnlyDb) + + removeDir(dbDir) + + test "Close multiple times": + var + dbDir = mkdtemp() + db = initMyDb(dbDir) + check not db.db.isNil + + db.close() + check db.db.isNil + + db.close() + check db.db.isNil + + removeDir(dbDir) + + test "Unknown column family": + const CF_UNKNOWN = "unknown" + + var + dbDir = mkdtemp() + db = initMyDb(dbDir, columnFamilyNames = @[CF_DEFAULT, CF_OTHER]) - var e1 = db.rocksdb.contains(key) - check e1.isok and e1.value == true + let r = db.put(key, val, CF_UNKNOWN) + check r.isErr() and r.error() == "rocksdb: unknown column family" - var e2 = db.rocksdb.contains(otherKey) - check e2.isok and e2.value == false + var bytes: seq[byte] + let r2 = db.get(key, proc(data: openArray[byte]) = bytes = @data, CF_UNKNOWN) + check r2.isErr() and r2.error() == "rocksdb: unknown column family" - var d = db.rocksdb.del(key) - check d.isok and d.value == true + let r3 = db.contains(key, CF_UNKNOWN) + check r3.isErr() and r3.error() == "rocksdb: unknown column family" - e1 = db.rocksdb.contains(key) - check e1.isok and e1.value == false + let r4 = db.del(key, CF_UNKNOWN) + check r4.isErr() and r4.error() == "rocksdb: unknown column family" - test() + db.close() + removeDir(dbDir) \ No newline at end of file diff --git a/tests/test_rocksdb_c.nim b/tests/test_rocksdb_c.nim index 63703cd..460da05 100644 --- a/tests/test_rocksdb_c.nim +++ b/tests/test_rocksdb_c.nim @@ -10,15 +10,16 @@ {.used.} import - cpuinfo, os, unittest, + std/[cpuinfo, os], tempfile, + unittest2, ../rocksdb suite "RocksDB C wrapper tests": setup: let - dbPath: cstring = mkdtemp() - dbBackupPath: cstring = mkdtemp() + dbPath = mkdtemp().cstring + dbBackupPath = mkdtemp().cstring teardown: removeDir($dbPath)