Column families support

README.md

@@ -10,7 +10,7 @@ A Nim wrapper for [Facebook's RocksDB](https://github.com/facebook/rocksdb), a p
 
 ## Current status
 
-Nim-RocksDB currently provides a wrapper for the low-level functions of RocksDB
+Nim-RocksDB provides a wrapper for the low-level functions in the librocksdb c library.
 
 ## Requirements
 
@@ -30,14 +30,6 @@ nim c -d:LibrocksbStaticArgs='-l:librocksdb.a' --gcc.linkerexe=g++ --threads:on
 
 (we need the C++ linker profile because it's a C++ library)
 
-## Future directions
-
-In the future, Nim-RocksDB might provide a high-level API that:
-
-- is more in line with Nim conventions (types in CamelCase),
-- automatically checks for errors,
-- leverage Nim features like destructors for automatic resource cleanup.
-
 ### Contribution
 
 Any contribution intentionally submitted for inclusion in the work by you shall be dual licensed as above, without any

rocksdb.nim

@@ -1,236 +1,12 @@
 # Nim-RocksDB
-# Copyright 2018 Status Research & Development GmbH
+# Copyright 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 #
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
 #  * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
 #
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
-{.push raises: [Defect].}
+import ./rocksdb/[backup, rocksdb]
 
-import cpuinfo, options, stew/[byteutils, results]
-
-from system/ansi_c import c_free
-
-export results
-
-const useCApi = true
-
-when useCApi:
-  import rocksdb/librocksdb
-  export librocksdb
-
-else:
-  {.error: "The C++ API of RocksDB is not supported yet".}
-
-  # The intention of this template is that it will hide the
-  # difference between the C and C++ APIs for objects such
-  # as Read/WriteOptions, which are allocated either on the
-  # stack or the heap.
-  template initResource(resourceName) =
-    var res = resourceName()
-    res
-
-type
-  RocksDBInstance* = object
-    db*: rocksdb_t
-    backupEngine: rocksdb_backup_engine_t
-    options*: rocksdb_options_t
-    readOptions*: rocksdb_readoptions_t
-    writeOptions: rocksdb_writeoptions_t
-    dbPath: string  # needed for clear()
-
-  DataProc* = proc(val: openArray[byte]) {.gcsafe, raises: [Defect].}
-
-  RocksDBResult*[T] = Result[T, string]
-
-template bailOnErrors {.dirty.} =
-  if not errors.isNil:
-    result.err($errors)
-    rocksdb_free(errors)
-    return
-
-proc init*(rocks: var RocksDBInstance,
-           dbPath, dbBackupPath: string,
-           readOnly = false,
-           cpus = countProcessors(),
-           createIfMissing = true,
-           maxOpenFiles = -1): RocksDBResult[void] =
-  rocks.options = rocksdb_options_create()
-  rocks.readOptions = rocksdb_readoptions_create()
-  rocks.writeOptions = rocksdb_writeoptions_create()
-  rocks.dbPath = dbPath
-
-  # Optimize RocksDB. This is the easiest way to get RocksDB to perform well:
-  rocksdb_options_increase_parallelism(rocks.options, cpus.int32)
-  # This requires snappy - disabled because rocksdb is not always compiled with
-  # snappy support (for example Fedora 28, certain Ubuntu versions)
-  # rocksdb_options_optimize_level_style_compaction(options, 0);
-  rocksdb_options_set_create_if_missing(rocks.options, uint8(createIfMissing))
-  # default set to keep all files open (-1), allow setting it to a specific
-  # value, e.g. in case the application limit would be reached.
-  rocksdb_options_set_max_open_files(rocks.options, maxOpenFiles.cint)
-
-  var errors: cstring
-  if readOnly:
-    rocks.db = rocksdb_open_for_read_only(rocks.options, dbPath, 0'u8, errors.addr)
-  else:
-    rocks.db = rocksdb_open(rocks.options, dbPath, errors.addr)
-  bailOnErrors()
-  rocks.backupEngine = rocksdb_backup_engine_open(rocks.options,
-                                                  dbBackupPath, errors.addr)
-  bailOnErrors()
-  ok()
-
-template initRocksDB*(args: varargs[untyped]): Option[RocksDBInstance] =
-  var db: RocksDBInstance
-  if not init(db, args):
-    none(RocksDBInstance)
-  else:
-    some(db)
-
-template getImpl(T: type) {.dirty.} =
-  if key.len <= 0:
-    return err("rocksdb: key cannot be empty on get")
-
-  var
-    errors: cstring
-    len: csize_t
-    data = rocksdb_get(db.db, db.readOptions,
-                       cast[cstring](unsafeAddr key[0]), csize_t(key.len),
-                       addr len, addr errors)
-  bailOnErrors()
-  if not data.isNil:
-    result = ok(toOpenArray(data, 0, int(len) - 1).to(T))
-    rocksdb_free(data)
-  else:
-    result = err("")
-
-proc get*(db: RocksDBInstance, key: openArray[byte], onData: DataProc): RocksDBResult[bool] =
-  if key.len <= 0:
-    return err("rocksdb: key cannot be empty on get")
-
-  var
-    errors: cstring
-    len: csize_t
-    data = rocksdb_get(db.db, db.readOptions,
-                       cast[cstring](unsafeAddr key[0]), csize_t(key.len),
-                       addr len, addr errors)
-  bailOnErrors()
-  if not data.isNil:
-    # TODO onData may raise a Defect - in theory we could catch it and free the
-    #      memory but this has a small overhead - setjmp (C) or RTTI (C++) -
-    #      reconsider this once the exception dust settles
-    onData(toOpenArrayByte(data, 0, int(len) - 1))
-    rocksdb_free(data)
-    ok(true)
-  else:
-    ok(false)
-
-proc get*(db: RocksDBInstance, key: openArray[byte]): RocksDBResult[string] {.deprecated: "DataProc".} =
-  ## Get value for `key`. If no value exists, set `result.ok` to `false`,
-  ## and result.error to `""`.
-  var res: RocksDBResult[string]
-  proc onData(data: openArray[byte]) =
-    res.ok(string.fromBytes(data))
-
-  if ? db.get(key, onData):
-    res
-  else:
-    ok("")
-
-proc getBytes*(db: RocksDBInstance, key: openArray[byte]): RocksDBResult[seq[byte]]  {.deprecated: "DataProc".} =
-  ## Get value for `key`. If no value exists, set `result.ok` to `false`,
-  ## and result.error to `""`.
-  var res: RocksDBResult[seq[byte]]
-  proc onData(data: openArray[byte]) =
-    res.ok(@data)
-
-  if ? db.get(key, onData):
-    res
-  else:
-    err("")
-
-proc put*(db: RocksDBInstance, key, val: openArray[byte]): RocksDBResult[void] =
-  if key.len <= 0:
-    return err("rocksdb: key cannot be empty on put")
-
-  var
-    errors: cstring
-
-  rocksdb_put(db.db, db.writeOptions,
-              cast[cstring](unsafeAddr key[0]), csize_t(key.len),
-              cast[cstring](if val.len > 0: unsafeAddr val[0] else: nil),
-              csize_t(val.len),
-              errors.addr)
-
-  bailOnErrors()
-  ok()
-
-proc contains*(db: RocksDBInstance, key: openArray[byte]): RocksDBResult[bool] =
-  if key.len <= 0:
-    return err("rocksdb: key cannot be empty on contains")
-
-  var
-    errors: cstring
-    len: csize_t
-    data = rocksdb_get(db.db, db.readOptions,
-                       cast[cstring](unsafeAddr key[0]), csize_t(key.len),
-                       addr len, errors.addr)
-  bailOnErrors()
-  if not data.isNil:
-    rocksdb_free(data)
-    ok(true)
-  else:
-    ok(false)
-
-proc del*(db: RocksDBInstance, key: openArray[byte]): RocksDBResult[bool] =
-  if key.len <= 0:
-    return err("rocksdb: key cannot be empty on del")
-
-  # This seems like a bad idea, but right now I don't want to
-  # get sidetracked by this. --Adam
-  if not db.contains(key).get:
-    return ok(false)
-
-  var errors: cstring
-  rocksdb_delete(db.db, db.writeOptions,
-                  cast[cstring](unsafeAddr key[0]), csize_t(key.len),
-                  errors.addr)
-  bailOnErrors()
-  ok(true)
-
-proc clear*(db: var RocksDBInstance): RocksDBResult[bool] =
-  raiseAssert "unimplemented"
-
-proc backup*(db: RocksDBInstance): RocksDBResult[void] =
-  var errors: cstring
-  rocksdb_backup_engine_create_new_backup(db.backupEngine, db.db, errors.addr)
-  bailOnErrors()
-  ok()
-
-# XXX: destructors are just too buggy at the moment:
-# https://github.com/nim-lang/Nim/issues/8112
-# proc `=destroy`*(db: var RocksDBInstance) =
-proc close*(db: var RocksDBInstance) =
-  template freeField(name) =  
-    type FieldType = typeof db.`name`
-    if db.`name`.isNil:
-      `rocksdb name destroy`(db.`name`)
-      db.`name` = FieldType(nil)
-  template setFieldToNil(name) =
-    type FieldType = typeof db.`name`
-    db.`name` = FieldType(nil)
-
-  freeField(writeOptions)
-  freeField(readOptions)
-  freeField(options)
-
-  if not db.backupEngine.isNil:
-    rocksdb_backup_engine_close(db.backupEngine)
-    setFieldToNil(backupEngine)
-
-  if not db.db.isNil:
-    rocksdb_close(db.db)
-    setFieldToNil(db)
+export backup, rocksdb

rocksdb.nimble

@@ -1,15 +1,16 @@
 packageName   = "rocksdb"
-version       = "0.3.1"
+version       = "0.4.0"
 author        = "Status Research & Development GmbH"
 description   = "A wrapper for Facebook's RocksDB, an embeddable, persistent key-value store for fast storage"
 license       = "Apache License 2.0 or GPLv2"
 skipDirs      = @["examples", "tests"]
 mode          = ScriptMode.Verbose
 
 ### Dependencies
-requires "nim >= 1.2.0",
+requires "nim >= 1.6.0",
          "stew",
-         "tempfile"
+         "tempfile",
+         "unittest2"
 
 proc test(args, path: string) =
   if not dirExists "build":
@@ -18,7 +19,7 @@ proc test(args, path: string) =
     " --outdir:build -r --hints:off --threads:on --skipParentCfg " & path
 
 task test, "Run tests":
-  test "", "tests/all.nim"
+  test "", "tests/test_all.nim"
   # Too troublesome to install "librocksdb.a" in CI, but this is how we would
   # test it (we need the C++ linker profile because it's a C++ library):
   # test "-d:LibrocksbStaticArgs='-l:librocksdb.a' --gcc.linkerexe=g++", "tests/all.nim"

rocksdb/backup.nim

@@ -0,0 +1,93 @@
+# Nim-RocksDB
+# Copyright 2024 Status Research & Development GmbH
+# Licensed under either of
+#
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+#  * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
+#
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+
+import
+  ./lib/librocksdb,
+  ./internal/utils,
+  ./options/backupopts,
+  ./rocksdb
+
+export
+  results,
+  backupopts,
+  rocksdb
+
+type
+  BackupEnginePtr* = ptr rocksdb_backup_engine_t
+
+  BackupEngineRef* = ref object
+    cPtr: BackupEnginePtr
+    path: string
+    backupOpts: BackupEngineOptionsRef
+
+proc openBackupEngine*(
+    path: string,
+    backupOpts = defaultBackupEngineOptions()): RocksDBResult[BackupEngineRef] =
+
+  var errors: cstring
+  let backupEnginePtr = rocksdb_backup_engine_open(
+    backupOpts.cPtr,
+    path.cstring,
+    cast[cstringArray](errors.addr))
+  bailOnErrors(errors)
+
+  let engine = BackupEngineRef(
+    cPtr: backupEnginePtr,
+    path: path,
+    backupOpts: backupOpts)
+  ok(engine)
+
+template isClosed*(backupEngine: BackupEngineRef): bool =
+  backupEngine.cPtr.isNil()
+
+proc createNewBackup*(
+    backupEngine: BackupEngineRef,
+    db: RocksDbRef): RocksDBResult[void] =
+  doAssert not backupEngine.isClosed()
+
+  var errors: cstring
+  rocksdb_backup_engine_create_new_backup(
+    backupEngine.cPtr,
+    db.cPtr,
+    cast[cstringArray](errors.addr))
+  bailOnErrors(errors)
+
+  ok()
+
+proc restoreDbFromLatestBackup*(
+    backupEngine: BackupEngineRef,
+    dbDir: string,
+    walDir = dbDir,
+    keepLogFiles = false): RocksDBResult[void] =
+  doAssert not backupEngine.isClosed()
+
+  let restoreOptions = rocksdb_restore_options_create()
+  rocksdb_restore_options_set_keep_log_files(restoreOptions, keepLogFiles.cint)
+
+  var errors: cstring
+  rocksdb_backup_engine_restore_db_from_latest_backup(
+    backupEngine.cPtr,
+    dbDir.cstring,
+    walDir.cstring,
+    restoreOptions,
+    cast[cstringArray](errors.addr))
+  bailOnErrors(errors)
+
+  rocksdb_restore_options_destroy(restoreOptions)
+
+  ok()
+
+proc close*(backupEngine: var BackupEngineRef) =
+  if not backupEngine.isClosed():
+    rocksdb_backup_engine_close(backupEngine.cPtr)
+    backupEngine.cPtr = nil
+
+