From aaaa2bebc67dd39e6e03a41f231422f125b94e92 Mon Sep 17 00:00:00 2001 From: web3-developer <51288821+web3-developer@users.noreply.github.com> Date: Wed, 3 Jul 2024 16:14:28 +0800 Subject: [PATCH] Add additional opts getters and setters for ReadOptionsRef and WriteOptionsRef. Updated BackupEngineOptionsRef to use newer c library type. --- rocksdb/backup.nim | 14 +++++++-- rocksdb/options/backupopts.nim | 48 ++++++++++++++++++++----------- rocksdb/options/dbopts.nim | 18 ++++++------ rocksdb/options/readopts.nim | 27 +++++++++++++++-- rocksdb/options/writeopts.nim | 27 +++++++++++++++-- tests/options/test_backupopts.nim | 6 ++-- tests/test_backup.nim | 5 ++-- tests/test_rocksdb.nim | 22 ++++++++++++++ 8 files changed, 129 insertions(+), 38 deletions(-) diff --git a/rocksdb/backup.nim b/rocksdb/backup.nim index 98301b4..2c251d9 100644 --- a/rocksdb/backup.nim +++ b/rocksdb/backup.nim @@ -18,14 +18,16 @@ export backupopts, rocksdb, rocksresult type BackupEnginePtr* = ptr rocksdb_backup_engine_t + EnvPtr = ptr rocksdb_env_t BackupEngineRef* = ref object cPtr: BackupEnginePtr + env: EnvPtr path: string backupOpts: BackupEngineOptionsRef proc openBackupEngine*( - path: string, backupOpts = defaultBackupEngineOptions(autoClose = true) + path: string, backupOpts = defaultBackupEngineOptions(path, autoClose = true) ): RocksDBResult[BackupEngineRef] = ## Create a new backup engine. The `path` parameter is the path of the backup ## directory. Note that the same directory should not be used for both backups @@ -35,12 +37,14 @@ proc openBackupEngine*( ## default backup options will be closed when the backup engine is closed. ## If `backupOpts` are provided, they will need to be closed manually. + let env = rocksdb_create_default_env() var errors: cstring - let backupEnginePtr = rocksdb_backup_engine_open( - backupOpts.cPtr, path.cstring, cast[cstringArray](errors.addr) + let backupEnginePtr = rocksdb_backup_engine_open_opts( + backupOpts.cPtr, env, cast[cstringArray](errors.addr) ) bailOnErrorsWithCleanup(errors): autoCloseNonNil(backupOpts) + rocksdb_env_destroy(env) let engine = BackupEngineRef(cPtr: backupEnginePtr, path: path, backupOpts: backupOpts) @@ -94,4 +98,8 @@ proc close*(backupEngine: BackupEngineRef) = rocksdb_backup_engine_close(backupEngine.cPtr) backupEngine.cPtr = nil + if not backupEngine.env.isNil(): + rocksdb_env_destroy(backupEngine.env) + backupEngine.env = nil + autoCloseNonNil(backupEngine.backupOpts) diff --git a/rocksdb/options/backupopts.nim b/rocksdb/options/backupopts.nim index 7d01294..bf3c1c8 100644 --- a/rocksdb/options/backupopts.nim +++ b/rocksdb/options/backupopts.nim @@ -12,32 +12,46 @@ import ../lib/librocksdb type - BackupEngineOptionsPtr* = ptr rocksdb_options_t + BackupEngineOptionsPtr* = ptr rocksdb_backup_engine_options_t BackupEngineOptionsRef* = ref object cPtr: BackupEngineOptionsPtr autoClose*: bool # if true then close will be called when the backup engine is closed -proc createBackupEngineOptions*(autoClose = false): BackupEngineOptionsRef = - BackupEngineOptionsRef(cPtr: rocksdb_options_create(), autoClose: autoClose) +proc createBackupEngineOptions*( + backupDir: string, autoClose = false +): BackupEngineOptionsRef = + BackupEngineOptionsRef( + cPtr: rocksdb_backup_engine_options_create(backupDir.cstring), autoClose: autoClose + ) -proc isClosed*(engineOpts: BackupEngineOptionsRef): bool {.inline.} = - engineOpts.cPtr.isNil() +proc isClosed*(backupOpts: BackupEngineOptionsRef): bool {.inline.} = + backupOpts.cPtr.isNil() -proc cPtr*(engineOpts: BackupEngineOptionsRef): BackupEngineOptionsPtr = - doAssert not engineOpts.isClosed() - engineOpts.cPtr +proc cPtr*(backupOpts: BackupEngineOptionsRef): BackupEngineOptionsPtr = + doAssert not backupOpts.isClosed() + backupOpts.cPtr -# TODO: Add setters and getters for backup options properties. +# template opt(nname, ntyp, ctyp: untyped) = +# proc `nname=`*(backupOpts: BackupEngineOptionsRef, value: ntyp) = +# doAssert not backupOpts.isClosed() +# `rocksdb_options_set nname`(backupOpts.cPtr, value.ctyp) -proc defaultBackupEngineOptions*(autoClose = false): BackupEngineOptionsRef {.inline.} = - let opts = createBackupEngineOptions(autoClose) +# proc `nname`*(backupOpts: BackupEngineOptionsRef): ntyp = +# doAssert not backupOpts.isClosed() +# ntyp `rocksdb_options_get nname`(backupOpts.cPtr) - # TODO: set defaults here +# opt shareTableFiles, bool, uint8 + +proc defaultBackupEngineOptions*( + backupDir: string, autoClose = false +): BackupEngineOptionsRef {.inline.} = + let backupOpts = createBackupEngineOptions(backupDir, autoClose) - opts + # TODO: set defaults here + backupOpts -proc close*(engineOpts: BackupEngineOptionsRef) = - if not engineOpts.isClosed(): - rocksdb_options_destroy(engineOpts.cPtr) - engineOpts.cPtr = nil +proc close*(backupOpts: BackupEngineOptionsRef) = + if not backupOpts.isClosed(): + rocksdb_backup_engine_options_destroy(backupOpts.cPtr) + backupOpts.cPtr = nil diff --git a/rocksdb/options/dbopts.nim b/rocksdb/options/dbopts.nim index 2349f08..3bc3ae3 100644 --- a/rocksdb/options/dbopts.nim +++ b/rocksdb/options/dbopts.nim @@ -40,11 +40,11 @@ proc increaseParallelism*(dbOpts: DbOptionsRef, totalThreads: int) = template opt(nname, ntyp, ctyp: untyped) = proc `nname=`*(dbOpts: DbOptionsRef, value: ntyp) = - doAssert not dbOpts.isClosed + doAssert not dbOpts.isClosed() `rocksdb_options_set nname`(dbOpts.cPtr, value.ctyp) proc `nname`*(dbOpts: DbOptionsRef): ntyp = - doAssert not dbOpts.isClosed + doAssert not dbOpts.isClosed() ntyp `rocksdb_options_get nname`(dbOpts.cPtr) opt createIfMissing, bool, uint8 @@ -102,22 +102,22 @@ proc `rowCache=`*(dbOpts: DbOptionsRef, cache: CacheRef) = dbOpts.cache = cache proc defaultDbOptions*(autoClose = false): DbOptionsRef = - let opts: DbOptionsRef = createDbOptions(autoClose) + let dbOpts: DbOptionsRef = createDbOptions(autoClose) # Optimize RocksDB. This is the easiest way to get RocksDB to perform well: - opts.increaseParallelism(countProcessors()) - opts.createIfMissing = true + dbOpts.increaseParallelism(countProcessors()) + dbOpts.createIfMissing = true # Enable creating column families if they do not exist - opts.createMissingColumnFamilies = true + dbOpts.createMissingColumnFamilies = true # Options recommended by rocksdb devs themselves, for new databases # https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning#other-general-options - opts.maxBackgroundJobs = 6 - opts.bytesPerSync = 1048576 + dbOpts.maxBackgroundJobs = 6 + dbOpts.bytesPerSync = 1048576 - opts + dbOpts proc close*(dbOpts: DbOptionsRef) = if not dbOpts.isClosed(): diff --git a/rocksdb/options/readopts.nim b/rocksdb/options/readopts.nim index f60269e..976836f 100644 --- a/rocksdb/options/readopts.nim +++ b/rocksdb/options/readopts.nim @@ -28,11 +28,34 @@ proc cPtr*(readOpts: ReadOptionsRef): ReadOptionsPtr = doAssert not readOpts.isClosed() readOpts.cPtr -# TODO: Add setters and getters for read options properties. +template opt(nname, ntyp, ctyp: untyped) = + proc `nname=`*(readOpts: ReadOptionsRef, value: ntyp) = + doAssert not readOpts.isClosed() + `rocksdb_readoptions_set nname`(readOpts.cPtr, value.ctyp) + + proc `nname`*(readOpts: ReadOptionsRef): ntyp = + doAssert not readOpts.isClosed() + ntyp `rocksdb_readoptions_get nname`(readOpts.cPtr) + +opt verifyChecksums, bool, uint8 +opt fillCache, bool, uint8 +opt readTier, int, cint +opt tailing, bool, uint8 +opt totalOrderSeek, bool, uint8 +opt prefixSameAsStart, bool, uint8 +opt pinData, bool, uint8 +opt backgroundPurgeOnIteratorCleanup, bool, uint8 +opt readaheadSize, int, csize_t +opt maxSkippableInternalKeys, int, csize_t +opt ignoreRangeDeletions, bool, uint8 +opt deadline, int, uint64 +opt ioTimeout, int, uint64 proc defaultReadOptions*(autoClose = false): ReadOptionsRef {.inline.} = - createReadOptions(autoClose) + let readOpts = createReadOptions(autoClose) + # TODO: set prefered defaults + readOpts proc close*(readOpts: ReadOptionsRef) = if not readOpts.isClosed(): diff --git a/rocksdb/options/writeopts.nim b/rocksdb/options/writeopts.nim index f621e7f..62c3f48 100644 --- a/rocksdb/options/writeopts.nim +++ b/rocksdb/options/writeopts.nim @@ -28,11 +28,34 @@ proc cPtr*(writeOpts: WriteOptionsRef): WriteOptionsPtr = doAssert not writeOpts.isClosed() writeOpts.cPtr -# TODO: Add setters and getters for write options properties. +template opt(nname, ntyp, ctyp: untyped) = + proc `nname=`*(writeOpts: WriteOptionsRef, value: ntyp) = + doAssert not writeOpts.isClosed() + `rocksdb_writeoptions_set nname`(writeOpts.cPtr, value.ctyp) + + proc `nname`*(writeOpts: WriteOptionsRef): ntyp = + doAssert not writeOpts.isClosed() + ntyp `rocksdb_writeoptions_get nname`(writeOpts.cPtr) + +opt sync, bool, uint8 +opt ignoreMissingColumnFamilies, bool, uint8 +opt noSlowdown, bool, uint8 +opt lowPri, bool, uint8 +opt memtableInsertHintPerBatch, bool, uint8 + +proc `disableWAL=`*(writeOpts: WriteOptionsRef, value: bool) = + doAssert not writeOpts.isClosed() + rocksdb_writeoptions_disable_WAL(writeOpts.cPtr, value.cint) + +proc disableWAL*(writeOpts: WriteOptionsRef): bool = + doAssert not writeOpts.isClosed() + rocksdb_writeoptions_get_disable_WAL(writeOpts.cPtr).bool proc defaultWriteOptions*(autoClose = false): WriteOptionsRef {.inline.} = - createWriteOptions(autoClose) + let writeOpts = createWriteOptions(autoClose) + # TODO: set prefered defaults + writeOpts proc close*(writeOpts: WriteOptionsRef) = if not writeOpts.isClosed(): diff --git a/tests/options/test_backupopts.nim b/tests/options/test_backupopts.nim index 9d0e21a..7f9912f 100644 --- a/tests/options/test_backupopts.nim +++ b/tests/options/test_backupopts.nim @@ -13,21 +13,21 @@ import unittest2, ../../rocksdb/options/backupopts suite "BackupEngineOptionsRef Tests": test "Test newBackupEngineOptions": - var backupOpts = createBackupEngineOptions() + let backupOpts = createBackupEngineOptions(".") check not backupOpts.cPtr.isNil() backupOpts.close() test "Test defaultBackupEngineOptions": - var backupOpts = defaultBackupEngineOptions() + let backupOpts = defaultBackupEngineOptions() check not backupOpts.cPtr.isNil() backupOpts.close() test "Test close": - var backupOpts = defaultBackupEngineOptions() + let backupOpts = defaultBackupEngineOptions() check not backupOpts.isClosed() backupOpts.close() diff --git a/tests/test_backup.nim b/tests/test_backup.nim index 4e50650..85fb42d 100644 --- a/tests/test_backup.nim +++ b/tests/test_backup.nim @@ -28,6 +28,7 @@ suite "BackupEngineRef Tests": db.close() removeDir($dbPath) removeDir($dbBackupPath) + removeDir($dbRestorePath) test "Test backup": var engine = initBackupEngine(dbBackupPath) @@ -62,7 +63,7 @@ suite "BackupEngineRef Tests": test "Test auto close enabled": let - backupOpts = defaultBackupEngineOptions(autoClose = true) + backupOpts = defaultBackupEngineOptions(dbPath, autoClose = true) backupEngine = openBackupEngine(dbPath, backupOpts).get() check: @@ -77,7 +78,7 @@ suite "BackupEngineRef Tests": test "Test auto close disabled": let - backupOpts = defaultBackupEngineOptions(autoClose = false) + backupOpts = defaultBackupEngineOptions(dbPath, autoClose = false) backupEngine = openBackupEngine(dbPath, backupOpts).get() check: diff --git a/tests/test_rocksdb.nim b/tests/test_rocksdb.nim index 160609a..71ae300 100644 --- a/tests/test_rocksdb.nim +++ b/tests/test_rocksdb.nim @@ -407,3 +407,25 @@ suite "RocksDbRef Tests": writeOpts.isClosed() == false columnFamilies[0].isClosed() == false db.isClosed() == true + + test "Test compression libraries linked": + let + dbPath = mkdtemp() / "compression" + cfOpts = defaultColFamilyOptions(autoClose = false) + cfDescriptor = initColFamilyDescriptor(CF_DEFAULT, cfOpts) + + cfOpts.compression = lz4Compression + check cfOpts.compression == lz4Compression + let cfDescriptor = initColFamilyDescriptor(CF_DEFAULT, cfOpts) + let db1 = openRocksDb(dbPath, columnFamilies = @[cfDescriptor]).get() + check db1.put(key, val).isOk() + db1.close() + + cfOpts.compression = zstdCompression + check cfOpts.compression == zstdCompression + let db2 = openRocksDb(dbPath, columnFamilies = @[cfDescriptor]).get() + check db2.put(key, val).isOk() + db2.close() + + cfOpts.close() + removeDir($dbPath)