From ee15ce027b27d5b44b8e2bd3b3d6227568ec0803 Mon Sep 17 00:00:00 2001 From: web3-developer <51288821+web3-developer@users.noreply.github.com> Date: Thu, 27 Jun 2024 01:31:39 +0800 Subject: [PATCH] Update API to support passing in ColFamilyHandleRef instead of column family string. (#52) * Refactor API to pass in column family handle instead of string. * Check for unknown column family in getColFamilyHandle and update tests. * Update column family function names. Remove redundant exists check. --- .github/workflows/ci.yml | 4 +- rocksdb.nimble | 2 +- rocksdb/columnfamily.nim | 41 +++++++------ rocksdb/internal/cftable.nim | 2 +- rocksdb/rocksdb.nim | 63 +++++++++----------- rocksdb/transactiondb.nim | 18 +++++- rocksdb/transactions/transaction.nim | 31 +++------- rocksdb/writebatch.nim | 27 ++------- tests/test_columnfamily.nim | 4 +- tests/test_helper.nim | 4 +- tests/test_rocksdb.nim | 83 +++++++++++--------------- tests/test_rocksiterator.nim | 33 +++++------ tests/test_sstfilewriter.nim | 16 ++--- tests/test_transactiondb.nim | 75 ++++++++++++------------ tests/test_writebatch.nim | 87 ++++++++-------------------- 15 files changed, 208 insertions(+), 282 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ada03a..126c8c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,11 +42,11 @@ jobs: include: - target: os: linux - builder: ubuntu-20.04 + builder: ubuntu-latest shell: bash - target: os: macos - builder: macos-11 + builder: macos-12 shell: bash - target: os: windows diff --git a/rocksdb.nimble b/rocksdb.nimble index 4b52ba5..bd266dc 100644 --- a/rocksdb.nimble +++ b/rocksdb.nimble @@ -1,5 +1,5 @@ packageName = "rocksdb" -version = "0.4.0" +version = "0.5.0" author = "Status Research & Development GmbH" description = "A wrapper for Facebook's RocksDB, an embeddable, persistent key-value store for fast storage" diff --git a/rocksdb/columnfamily.nim b/rocksdb/columnfamily.nim index 7862684..2e4a09f 100644 --- a/rocksdb/columnfamily.nim +++ b/rocksdb/columnfamily.nim @@ -20,6 +20,7 @@ {.push raises: [].} import ./rocksdb +import ./columnfamily/cfhandle, ./rocksdb export rocksdb @@ -27,34 +28,30 @@ type ColFamilyReadOnly* = object db: RocksDbReadOnlyRef name: string + handle: ColFamilyHandleRef ColFamilyReadWrite* = object db: RocksDbReadWriteRef name: string + handle: ColFamilyHandleRef -proc withColFamily*( +proc getColFamily*( db: RocksDbReadOnlyRef, name: string ): RocksDBResult[ColFamilyReadOnly] = ## Creates a new `ColFamilyReadOnly` from the given `RocksDbReadOnlyRef` and ## column family name. + doAssert not db.isClosed() - # validate that the column family exists - discard db.keyExists(@[0.byte], name).valueOr: - return err(error) + ok(ColFamilyReadOnly(db: db, name: name, handle: ?db.getColFamilyHandle(name))) - ok(ColFamilyReadOnly(db: db, name: name)) - -proc withColFamily*( +proc getColFamily*( db: RocksDbReadWriteRef, name: string ): RocksDBResult[ColFamilyReadWrite] = ## Create a new `ColFamilyReadWrite` from the given `RocksDbReadWriteRef` and ## column family name. + doAssert not db.isClosed() - # validate that the column family exists - discard db.keyExists(@[0.byte], name).valueOr: - return err(error) - - ok(ColFamilyReadWrite(db: db, name: name)) + ok(ColFamilyReadWrite(db: db, name: name, handle: ?db.getColFamilyHandle(name))) proc db*(cf: ColFamilyReadOnly | ColFamilyReadWrite): auto {.inline.} = ## Returns the underlying `RocksDbReadOnlyRef` or `RocksDbReadWriteRef`. @@ -64,46 +61,52 @@ proc name*(cf: ColFamilyReadOnly | ColFamilyReadWrite): string {.inline.} = ## Returns the name of the column family. cf.name +proc handle*( + cf: ColFamilyReadOnly | ColFamilyReadWrite +): ColFamilyHandleRef {.inline.} = + ## Returns the name of the column family. + cf.handle + proc get*( cf: ColFamilyReadOnly | ColFamilyReadWrite, key: openArray[byte], onData: DataProc ): RocksDBResult[bool] {.inline.} = ## Gets the value of the given key from the column family using the `onData` ## callback. - cf.db.get(key, onData, cf.name) + cf.db.get(key, onData, cf.handle) proc get*( cf: ColFamilyReadOnly | ColFamilyReadWrite, key: openArray[byte] ): RocksDBResult[seq[byte]] {.inline.} = ## Gets the value of the given key from the column family. - cf.db.get(key, cf.name) + cf.db.get(key, cf.handle) proc put*( cf: ColFamilyReadWrite, key, val: openArray[byte] ): RocksDBResult[void] {.inline.} = ## Puts a value for the given key into the column family. - cf.db.put(key, val, cf.name) + cf.db.put(key, val, cf.handle) proc keyExists*( cf: ColFamilyReadOnly | ColFamilyReadWrite, key: openArray[byte] ): RocksDBResult[bool] {.inline.} = ## Checks if the given key exists in the column family. - cf.db.keyExists(key, cf.name) + cf.db.keyExists(key, cf.handle) proc delete*( cf: ColFamilyReadWrite, key: openArray[byte] ): RocksDBResult[void] {.inline.} = ## Deletes the given key from the column family. - cf.db.delete(key, cf.name) + cf.db.delete(key, cf.handle) proc openIterator*( cf: ColFamilyReadOnly | ColFamilyReadWrite ): RocksDBResult[RocksIteratorRef] {.inline.} = ## Opens an `RocksIteratorRef` for the given column family. - cf.db.openIterator(cf.name) + cf.db.openIterator(cf.handle) proc openWriteBatch*(cf: ColFamilyReadWrite): WriteBatchRef {.inline.} = ## Opens a `WriteBatchRef` for the given column family. - cf.db.openWriteBatch(cf.name) + cf.db.openWriteBatch(cf.handle) proc write*( cf: ColFamilyReadWrite, updates: WriteBatchRef diff --git a/rocksdb/internal/cftable.nim b/rocksdb/internal/cftable.nim index 87d5d8b..32797d4 100644 --- a/rocksdb/internal/cftable.nim +++ b/rocksdb/internal/cftable.nim @@ -30,7 +30,7 @@ proc newColFamilyTable*( proc isClosed*(table: ColFamilyTableRef): bool {.inline.} = table.columnFamilies.isNil() -proc get*(table: ColFamilyTableRef, name: string): ColFamilyHandleRef = +proc get*(table: ColFamilyTableRef, name: string): ColFamilyHandleRef {.inline.} = table.columnFamilies.getOrDefault(name) proc close*(table: ColFamilyTableRef) = diff --git a/rocksdb/rocksdb.nim b/rocksdb/rocksdb.nim index afc933b..d012494 100644 --- a/rocksdb/rocksdb.nim +++ b/rocksdb/rocksdb.nim @@ -47,7 +47,7 @@ type path: string dbOpts: DbOptionsRef readOpts: ReadOptionsRef - defaultCfName: string + defaultCfHandle: ColFamilyHandleRef cfTable: ColFamilyTableRef RocksDbReadOnlyRef* = ref object of RocksDbRef @@ -143,6 +143,7 @@ proc openRocksDb*( dbOpts = useDbOpts # don't close on exit readOpts = (if readOpts.isNil: defaultReadOptions() else: readOpts) writeOpts = (if writeOpts.isNil: defaultWriteOptions() else: writeOpts) + cfTable = newColFamilyTable(cfNames.mapIt($it), cfHandles) db = RocksDbReadWriteRef( lock: createLock(), cPtr: rocksDbPtr, @@ -151,8 +152,8 @@ proc openRocksDb*( readOpts: readOpts, writeOpts: writeOpts, ingestOptsPtr: rocksdb_ingestexternalfileoptions_create(), - defaultCfName: DEFAULT_COLUMN_FAMILY_NAME, - cfTable: newColFamilyTable(cfNames.mapIt($it), cfHandles), + defaultCfHandle: cfTable.get(DEFAULT_COLUMN_FAMILY_NAME), + cfTable: cfTable, ) ok(db) @@ -198,17 +199,27 @@ proc openRocksDbReadOnly*( let dbOpts = useDbOpts # don't close on exit readOpts = (if readOpts.isNil: defaultReadOptions() else: readOpts) + cfTable = newColFamilyTable(cfNames.mapIt($it), cfHandles) db = RocksDbReadOnlyRef( lock: createLock(), cPtr: rocksDbPtr, path: path, dbOpts: dbOpts, readOpts: readOpts, - defaultCfName: DEFAULT_COLUMN_FAMILY_NAME, - cfTable: newColFamilyTable(cfNames.mapIt($it), cfHandles), + defaultCfHandle: cfTable.get(DEFAULT_COLUMN_FAMILY_NAME), + cfTable: cfTable, ) ok(db) +proc getColFamilyHandle*( + db: RocksDbRef, name: string +): RocksDBResult[ColFamilyHandleRef] = + let cfHandle = db.cfTable.get(name) + if cfHandle.isNil(): + err("rocksdb: unknown column family") + else: + ok(cfHandle) + proc isClosed*(db: RocksDbRef): bool {.inline.} = ## Returns `true` if the database has been closed and `false` otherwise. db.cPtr.isNil() @@ -222,7 +233,7 @@ proc get*( db: RocksDbRef, key: openArray[byte], onData: DataProc, - columnFamily = db.defaultCfName, + cfHandle = db.defaultCfHandle, ): RocksDBResult[bool] = ## Get the value for the given key from the specified column family. ## If the value does not exist, `false` will be returned in the result @@ -234,10 +245,6 @@ proc get*( if key.len() == 0: return err("rocksdb: key is empty") - let cfHandle = db.cfTable.get(columnFamily) - if cfHandle.isNil(): - return err("rocksdb: unknown column family") - var len: csize_t errors: cstring @@ -261,7 +268,7 @@ proc get*( ok(true) proc get*( - db: RocksDbRef, key: openArray[byte], columnFamily = db.defaultCfName + db: RocksDbRef, key: openArray[byte], cfHandle = db.defaultCfHandle ): RocksDBResult[seq[byte]] = ## Get the value for the given key from the specified column family. ## If the value does not exist, an empty error will be returned in the result. @@ -271,24 +278,20 @@ proc get*( proc onData(data: openArray[byte]) = dataRes.ok(@data) - let res = db.get(key, onData, columnFamily) + let res = db.get(key, onData, cfHandle) if res.isOk(): return dataRes dataRes.err(res.error()) proc put*( - db: RocksDbReadWriteRef, key, val: openArray[byte], columnFamily = db.defaultCfName + db: RocksDbReadWriteRef, key, val: openArray[byte], cfHandle = db.defaultCfHandle ): RocksDBResult[void] = ## Put the value for the given key into the specified column family. if key.len() == 0: return err("rocksdb: key is empty") - let cfHandle = db.cfTable.get(columnFamily) - if cfHandle.isNil(): - return err("rocksdb: unknown column family") - var errors: cstring rocksdb_put_cf( db.cPtr, @@ -309,7 +312,7 @@ proc put*( ok() proc keyExists*( - db: RocksDbRef, key: openArray[byte], columnFamily = db.defaultCfName + db: RocksDbRef, key: openArray[byte], cfHandle = db.defaultCfHandle ): RocksDBResult[bool] = ## Check if the key exists in the specified column family. ## Returns a result containing `true` if the key exists or a result @@ -323,11 +326,11 @@ proc keyExists*( proc(data: openArray[byte]) = discard , - columnFamily, + cfHandle, ) proc delete*( - db: RocksDbReadWriteRef, key: openArray[byte], columnFamily = db.defaultCfName + db: RocksDbReadWriteRef, key: openArray[byte], cfHandle = db.defaultCfHandle ): RocksDBResult[void] = ## Delete the value for the given key from the specified column family. ## If the value does not exist, the delete will be a no-op. @@ -336,10 +339,6 @@ proc delete*( if key.len() == 0: return err("rocksdb: key is empty") - let cfHandle = db.cfTable.get(columnFamily) - if cfHandle.isNil: - return err("rocksdb: unknown column family") - var errors: cstring rocksdb_delete_cf( db.cPtr, @@ -354,27 +353,23 @@ proc delete*( ok() proc openIterator*( - db: RocksDbRef, columnFamily = db.defaultCfName + db: RocksDbRef, cfHandle = db.defaultCfHandle ): RocksDBResult[RocksIteratorRef] = ## Opens an `RocksIteratorRef` for the specified column family. doAssert not db.isClosed() - let cfHandle = db.cfTable.get(columnFamily) - if cfHandle.isNil(): - return err("rocksdb: unknown column family") - let rocksIterPtr = rocksdb_create_iterator_cf(db.cPtr, db.readOpts.cPtr, cfHandle.cPtr) ok(newRocksIterator(rocksIterPtr)) proc openWriteBatch*( - db: RocksDbReadWriteRef, columnFamily = db.defaultCfName + db: RocksDbReadWriteRef, cfHandle = db.defaultCfHandle ): WriteBatchRef = ## Opens a `WriteBatchRef` which defaults to using the specified column family. doAssert not db.isClosed() - newWriteBatch(db.cfTable, columnFamily) + newWriteBatch(cfHandle) proc write*(db: RocksDbReadWriteRef, updates: WriteBatchRef): RocksDBResult[void] = ## Apply the updates in the `WriteBatchRef` to the database. @@ -389,17 +384,13 @@ proc write*(db: RocksDbReadWriteRef, updates: WriteBatchRef): RocksDBResult[void ok() proc ingestExternalFile*( - db: RocksDbReadWriteRef, filePath: string, columnFamily = db.defaultCfName + db: RocksDbReadWriteRef, filePath: string, cfHandle = db.defaultCfHandle ): RocksDBResult[void] = ## Ingest an external sst file into the database. The file will be ingested ## into the specified column family or the default column family if none is ## provided. doAssert not db.isClosed() - let cfHandle = db.cfTable.get(columnFamily) - if cfHandle.isNil(): - return err("rocksdb: unknown column family") - var sstPath = filePath.cstring errors: cstring diff --git a/rocksdb/transactiondb.nim b/rocksdb/transactiondb.nim index 79bfc2b..71f4dfc 100644 --- a/rocksdb/transactiondb.nim +++ b/rocksdb/transactiondb.nim @@ -36,6 +36,7 @@ type path: string dbOpts: DbOptionsRef txDbOpts: TransactionDbOptionsRef + defaultCfHandle: ColFamilyHandleRef cfTable: ColFamilyTableRef proc openTransactionDb*( @@ -79,16 +80,27 @@ proc openTransactionDb*( txDbOpts = (if txDbOpts.isNil: defaultTransactionDbOptions() else: txDbOpts ) + cfTable = newColFamilyTable(cfNames.mapIt($it), cfHandles) db = TransactionDbRef( lock: createLock(), cPtr: txDbPtr, path: path, dbOpts: dbOpts, txDbOpts: txDbOpts, - cfTable: newColFamilyTable(cfNames.mapIt($it), cfHandles), + defaultCfHandle: cfTable.get(DEFAULT_COLUMN_FAMILY_NAME), + cfTable: cfTable, ) ok(db) +proc getColFamilyHandle*( + db: TransactionDbRef, name: string +): RocksDBResult[ColFamilyHandleRef] = + let cfHandle = db.cfTable.get(name) + if cfHandle.isNil(): + err("rocksdb: unknown column family") + else: + ok(cfHandle) + proc isClosed*(db: TransactionDbRef): bool {.inline.} = ## Returns `true` if the `TransactionDbRef` has been closed. db.cPtr.isNil() @@ -99,7 +111,7 @@ proc beginTransaction*( writeOpts = WriteOptionsRef(nil), txDbOpts = TransactionDbOptionsRef(nil), txOpts = defaultTransactionOptions(), - columnFamily = DEFAULT_COLUMN_FAMILY_NAME, + cfHandle = db.defaultCfHandle, ): TransactionRef = ## Begin a new transaction against the database. The transaction will default ## to using the specified column family. If no column family is specified @@ -114,7 +126,7 @@ proc beginTransaction*( let txPtr = rocksdb_transaction_begin(db.cPtr, writeOpts.cPtr, txOpts.cPtr, nil) - newTransaction(txPtr, readOpts, writeOpts, txOpts, columnFamily, db.cfTable) + newTransaction(txPtr, readOpts, writeOpts, txOpts, cfHandle) proc close*(db: TransactionDbRef) = ## Close the `TransactionDbRef`. diff --git a/rocksdb/transactions/transaction.nim b/rocksdb/transactions/transaction.nim index b877736..dd86c27 100644 --- a/rocksdb/transactions/transaction.nim +++ b/rocksdb/transactions/transaction.nim @@ -36,24 +36,21 @@ type readOpts: ReadOptionsRef writeOpts: WriteOptionsRef txOpts: TransactionOptionsRef - defaultCfName: string - cfTable: ColFamilyTableRef + defaultCfHandle: ColFamilyHandleRef proc newTransaction*( cPtr: TransactionPtr, readOpts: ReadOptionsRef, writeOpts: WriteOptionsRef, txOpts: TransactionOptionsRef, - defaultCfName: string, - cfTable: ColFamilyTableRef, + defaultCfHandle: ColFamilyHandleRef, ): TransactionRef = TransactionRef( cPtr: cPtr, readOpts: readOpts, writeOpts: writeOpts, txOpts: txOpts, - defaultCfName: defaultCfName, - cfTable: cfTable, + defaultCfHandle: defaultCfHandle, ) proc isClosed*(tx: TransactionRef): bool {.inline.} = @@ -64,7 +61,7 @@ proc get*( tx: TransactionRef, key: openArray[byte], onData: DataProc, - columnFamily = tx.defaultCfName, + cfHandle = tx.defaultCfHandle, ): RocksDBResult[bool] = ## Get the value for a given key from the transaction using the provided ## `onData` callback. @@ -72,10 +69,6 @@ proc get*( if key.len() == 0: return err("rocksdb: key is empty") - let cfHandle = tx.cfTable.get(columnFamily) - if cfHandle.isNil(): - return err("rocksdb: unknown column family") - var len: csize_t errors: cstring @@ -99,7 +92,7 @@ proc get*( ok(true) proc get*( - tx: TransactionRef, key: openArray[byte], columnFamily = tx.defaultCfName + tx: TransactionRef, key: openArray[byte], cfHandle = tx.defaultCfHandle ): RocksDBResult[seq[byte]] = ## Get the value for a given key from the transaction. @@ -107,24 +100,20 @@ proc get*( proc onData(data: openArray[byte]) = dataRes.ok(@data) - let res = tx.get(key, onData, columnFamily) + let res = tx.get(key, onData, cfHandle) if res.isOk(): return dataRes dataRes.err(res.error()) proc put*( - tx: TransactionRef, key, val: openArray[byte], columnFamily = tx.defaultCfName + tx: TransactionRef, key, val: openArray[byte], cfHandle = tx.defaultCfHandle ): RocksDBResult[void] = ## Put the value for the given key into the transaction. if key.len() == 0: return err("rocksdb: key is empty") - let cfHandle = tx.cfTable.get(columnFamily) - if cfHandle.isNil(): - return err("rocksdb: unknown column family") - var errors: cstring rocksdb_transaction_put_cf( tx.cPtr, @@ -144,17 +133,13 @@ proc put*( ok() proc delete*( - tx: TransactionRef, key: openArray[byte], columnFamily = tx.defaultCfName + tx: TransactionRef, key: openArray[byte], cfHandle = tx.defaultCfHandle ): RocksDBResult[void] = ## Delete the value for the given key from the transaction. if key.len() == 0: return err("rocksdb: key is empty") - let cfHandle = tx.cfTable.get(columnFamily) - if cfHandle.isNil: - return err("rocksdb: unknown column family") - var errors: cstring rocksdb_transaction_delete_cf( tx.cPtr, diff --git a/rocksdb/writebatch.nim b/rocksdb/writebatch.nim index 7770fad..3a3672f 100644 --- a/rocksdb/writebatch.nim +++ b/rocksdb/writebatch.nim @@ -11,7 +11,7 @@ {.push raises: [].} -import ./lib/librocksdb, ./internal/[cftable, utils], ./rocksresult +import ./lib/librocksdb, ./internal/cftable, ./rocksresult export rocksresult @@ -20,13 +20,10 @@ type WriteBatchRef* = ref object cPtr: WriteBatchPtr - defaultCfName: string - cfTable: ColFamilyTableRef + defaultCfHandle: ColFamilyHandleRef -proc newWriteBatch*(cfTable: ColFamilyTableRef, defaultCfName: string): WriteBatchRef = - WriteBatchRef( - cPtr: rocksdb_writebatch_create(), defaultCfName: defaultCfName, cfTable: cfTable - ) +proc newWriteBatch*(defaultCfHandle: ColFamilyHandleRef): WriteBatchRef = + WriteBatchRef(cPtr: rocksdb_writebatch_create(), defaultCfHandle: defaultCfHandle) proc isClosed*(batch: WriteBatchRef): bool {.inline.} = ## Returns `true` if the `WriteBatchRef` has been closed and `false` otherwise. @@ -48,19 +45,13 @@ proc count*(batch: WriteBatchRef): int = rocksdb_writebatch_count(batch.cPtr).int proc put*( - batch: WriteBatchRef, - key, val: openArray[byte], - columnFamily = DEFAULT_COLUMN_FAMILY_NAME, + batch: WriteBatchRef, key, val: openArray[byte], cfHandle = batch.defaultCfHandle ): RocksDBResult[void] = ## Add a put operation to the write batch. if key.len() == 0: return err("rocksdb: key is empty") - let cfHandle = batch.cfTable.get(columnFamily) - if cfHandle.isNil: - return err("rocksdb: unknown column family") - rocksdb_writebatch_put_cf( batch.cPtr, cfHandle.cPtr, @@ -77,19 +68,13 @@ proc put*( ok() proc delete*( - batch: WriteBatchRef, - key: openArray[byte], - columnFamily = DEFAULT_COLUMN_FAMILY_NAME, + batch: WriteBatchRef, key: openArray[byte], cfHandle = batch.defaultCfHandle ): RocksDBResult[void] = ## Add a delete operation to the write batch. if key.len() == 0: return err("rocksdb: key is empty") - let cfHandle = batch.cfTable.get(columnFamily) - if cfHandle.isNil: - return err("rocksdb: unknown column family") - rocksdb_writebatch_delete_cf( batch.cPtr, cfHandle.cPtr, cast[cstring](unsafeAddr key[0]), csize_t(key.len) ) diff --git a/tests/test_columnfamily.nim b/tests/test_columnfamily.nim index 28f6829..475124b 100644 --- a/tests/test_columnfamily.nim +++ b/tests/test_columnfamily.nim @@ -31,7 +31,7 @@ suite "ColFamily Tests": removeDir($dbPath) test "Basic operations": - let r0 = db.withColFamily(CF_OTHER) + let r0 = db.getColFamily(CF_OTHER) check r0.isOk() let cf = r0.value() @@ -78,7 +78,7 @@ suite "ColFamily Tests": # Open database in read only mode block: - var res = initReadOnlyDb(dbPath).withColFamily(CF_DEFAULT) + var res = initReadOnlyDb(dbPath).getColFamily(CF_DEFAULT) check res.isOk() let readOnlyCf = res.value() diff --git a/tests/test_helper.nim b/tests/test_helper.nim index 940660b..b0eae61 100644 --- a/tests/test_helper.nim +++ b/tests/test_helper.nim @@ -42,7 +42,9 @@ proc initTransactionDb*( path: string, columnFamilyNames: openArray[string] = @[] ): TransactionDbRef = let res = openTransactionDb( - path, columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it)) + path, + txDbOpts = defaultTransactionDbOptions(), + columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it)), ) if res.isErr(): echo res.error() diff --git a/tests/test_rocksdb.nim b/tests/test_rocksdb.nim index ef45e1a..f6070a0 100644 --- a/tests/test_rocksdb.nim +++ b/tests/test_rocksdb.nim @@ -25,6 +25,8 @@ suite "RocksDbRef Tests": let dbPath = mkdtemp() / "data" db = initReadWriteDb(dbPath, columnFamilyNames = @[CF_DEFAULT, CF_OTHER]) + defaultCfHandle = db.getColFamilyHandle(CF_DEFAULT).get() + otherCfHandle = db.getColFamilyHandle(CF_OTHER).get() teardown: db.close() @@ -88,7 +90,7 @@ suite "RocksDbRef Tests": check readOnlyDb.isClosed() test "Basic operations - default column family": - var s = db.put(key, val, CF_DEFAULT) + var s = db.put(key, val, defaultCfHandle) check s.isOk() var bytes: seq[byte] @@ -97,14 +99,14 @@ suite "RocksDbRef Tests": proc(data: openArray[byte]) = bytes = @data , - CF_DEFAULT, + defaultCfHandle, )[] check not db.get( otherKey, proc(data: openArray[byte]) = bytes = @data , - CF_DEFAULT, + defaultCfHandle, )[] var r1 = db.get(key) @@ -114,43 +116,43 @@ suite "RocksDbRef Tests": # there's no error string for missing keys check r2.isOk() == false and r2.error.len == 0 - var e1 = db.keyExists(key, CF_DEFAULT) + var e1 = db.keyExists(key, defaultCfHandle) check e1.isOk() and e1.value == true - var e2 = db.keyExists(otherKey, CF_DEFAULT) + var e2 = db.keyExists(otherKey, defaultCfHandle) check e2.isOk() and e2.value == false - var d = db.delete(key, CF_DEFAULT) + var d = db.delete(key, defaultCfHandle) check d.isOk() - e1 = db.keyExists(key, CF_DEFAULT) + e1 = db.keyExists(key, defaultCfHandle) check e1.isOk() and e1.value == false - d = db.delete(otherKey, CF_DEFAULT) + d = db.delete(otherKey, defaultCfHandle) check d.isOk() - close(db) + db.close() check db.isClosed() # Open database in read only mode block: var readOnlyDb = initReadOnlyDb(dbPath, columnFamilyNames = @[CF_DEFAULT]) - r = readOnlyDb.keyExists(key, CF_DEFAULT) + r = readOnlyDb.keyExists(key) check r.isOk() and r.value == false # Won't compile as designed: - # var r2 = readOnlyDb.put(key, @[123.byte], CF_DEFAULT) + # var r2 = readOnlyDb.put(key, @[123.byte], defaultCfHandle) # check r2.isErr() readOnlyDb.close() check readOnlyDb.isClosed() test "Basic operations - multiple column families": - var s = db.put(key, val, CF_DEFAULT) + var s = db.put(key, val, defaultCfHandle) check s.isOk() - var s2 = db.put(otherKey, val, CF_OTHER) + var s2 = db.put(otherKey, val, otherCfHandle) check s2.isOk() var bytes: seq[byte] @@ -159,14 +161,14 @@ suite "RocksDbRef Tests": proc(data: openArray[byte]) = bytes = @data , - CF_DEFAULT, + defaultCfHandle, )[] check not db.get( otherKey, proc(data: openArray[byte]) = bytes = @data , - CF_DEFAULT, + defaultCfHandle, )[] var bytes2: seq[byte] @@ -175,40 +177,40 @@ suite "RocksDbRef Tests": proc(data: openArray[byte]) = bytes2 = @data , - CF_OTHER, + otherCfHandle, )[] check not db.get( key, proc(data: openArray[byte]) = bytes2 = @data , - CF_OTHER, + otherCfHandle, )[] - var e1 = db.keyExists(key, CF_DEFAULT) + var e1 = db.keyExists(key, defaultCfHandle) check e1.isOk() and e1.value == true - var e2 = db.keyExists(otherKey, CF_DEFAULT) + var e2 = db.keyExists(otherKey, defaultCfHandle) check e2.isOk() and e2.value == false - var e3 = db.keyExists(key, CF_OTHER) + var e3 = db.keyExists(key, otherCfHandle) check e3.isOk() and e3.value == false - var e4 = db.keyExists(otherKey, CF_OTHER) + var e4 = db.keyExists(otherKey, otherCfHandle) check e4.isOk() and e4.value == true - var d = db.delete(key, CF_DEFAULT) + var d = db.delete(key, defaultCfHandle) check d.isOk() - e1 = db.keyExists(key, CF_DEFAULT) + e1 = db.keyExists(key, defaultCfHandle) check e1.isOk() and e1.value == false - d = db.delete(otherKey, CF_DEFAULT) + d = db.delete(otherKey, defaultCfHandle) check d.isOk() - var d2 = db.delete(key, CF_OTHER) + var d2 = db.delete(key, otherCfHandle) check d2.isOk() - e3 = db.keyExists(key, CF_OTHER) + e3 = db.keyExists(key, otherCfHandle) check e3.isOk() and e3.value == false - d2 = db.delete(otherKey, CF_OTHER) + d2 = db.delete(otherKey, otherCfHandle) check d2.isOk() - d2 = db.delete(otherKey, CF_OTHER) + d2 = db.delete(otherKey, otherCfHandle) check d2.isOk() db.close() @@ -219,11 +221,11 @@ suite "RocksDbRef Tests": var readOnlyDb = initReadOnlyDb(dbPath, columnFamilyNames = @[CF_DEFAULT, CF_OTHER]) - var r = readOnlyDb.keyExists(key, CF_OTHER) + var r = readOnlyDb.keyExists(key, readOnlyDb.getColFamilyHandle(CF_OTHER).get()) check r.isOk() and r.value == false # Does not compile as designed: - # var r2 = readOnlyDb.put(key, @[123.byte], CF_OTHER) + # var r2 = readOnlyDb.put(key, @[123.byte], otherCfHandle) # check r2.isErr() readOnlyDb.close() @@ -238,25 +240,8 @@ suite "RocksDbRef Tests": test "Unknown column family": const CF_UNKNOWN = "unknown" - - let r = db.put(key, val, CF_UNKNOWN) - check r.isErr() and r.error() == "rocksdb: unknown column family" - - var bytes: seq[byte] - let r2 = db.get( - key, - proc(data: openArray[byte]) = - bytes = @data - , - CF_UNKNOWN, - ) - check r2.isErr() and r2.error() == "rocksdb: unknown column family" - - let r3 = db.keyExists(key, CF_UNKNOWN) - check r3.isErr() and r3.error() == "rocksdb: unknown column family" - - let r4 = db.delete(key, CF_UNKNOWN) - check r4.isErr() and r4.error() == "rocksdb: unknown column family" + let cfHandleRes = db.getColFamilyHandle(CF_UNKNOWN) + check cfHandleRes.isErr() and cfHandleRes.error() == "rocksdb: unknown column family" test "Test missing key and values": let diff --git a/tests/test_rocksiterator.nim b/tests/test_rocksiterator.nim index e422dac..348ed96 100644 --- a/tests/test_rocksiterator.nim +++ b/tests/test_rocksiterator.nim @@ -30,20 +30,23 @@ suite "RocksIteratorRef Tests": dbPath = mkdtemp() / "data" db = initReadWriteDb(dbPath, columnFamilyNames = @[CF_DEFAULT, CF_OTHER, CF_EMPTY]) + defaultCfHandle = db.getColFamilyHandle(CF_DEFAULT).get() + otherCfHandle = db.getColFamilyHandle(CF_OTHER).get() + emptyCfHandle = db.getColFamilyHandle(CF_EMPTY).get() doAssert db.put(key1, val1).isOk() doAssert db.put(key2, val2).isOk() doAssert db.put(key3, val3).isOk() - doAssert db.put(key1, val1, CF_OTHER).isOk() - doAssert db.put(key2, val2, CF_OTHER).isOk() - doAssert db.put(key3, val3, CF_OTHER).isOk() + doAssert db.put(key1, val1, otherCfHandle).isOk() + doAssert db.put(key2, val2, otherCfHandle).isOk() + doAssert db.put(key3, val3, otherCfHandle).isOk() teardown: db.close() removeDir($dbPath) test "Iterate forwards using default column family": - let res = db.openIterator(CF_DEFAULT) + let res = db.openIterator(defaultCfHandle) check res.isOk() var iter = res.get() @@ -69,7 +72,7 @@ suite "RocksIteratorRef Tests": check expected == byte(4) test "Iterate backwards using other column family": - let res = db.openIterator(CF_OTHER) + let res = db.openIterator(otherCfHandle) check res.isOk() var iter = res.get() @@ -103,12 +106,12 @@ suite "RocksIteratorRef Tests": iter.close() test "Open two iterators on the same column family": - let res1 = db.openIterator(CF_DEFAULT) + let res1 = db.openIterator(defaultCfHandle) check res1.isOk() var iter1 = res1.get() defer: iter1.close() - let res2 = db.openIterator(CF_DEFAULT) + let res2 = db.openIterator(defaultCfHandle) check res2.isOk() var iter2 = res2.get() defer: @@ -126,12 +129,12 @@ suite "RocksIteratorRef Tests": iter2.value() == @[byte(3)] test "Open two iterators on different column families": - let res1 = db.openIterator(CF_DEFAULT) + let res1 = db.openIterator(defaultCfHandle) check res1.isOk() var iter1 = res1.get() defer: iter1.close() - let res2 = db.openIterator(CF_OTHER) + let res2 = db.openIterator(otherCfHandle) check res2.isOk() var iter2 = res2.get() defer: @@ -148,14 +151,8 @@ suite "RocksIteratorRef Tests": iter2.key() == @[byte(3)] iter2.value() == @[byte(3)] - test "Invalid column family": - let res = db.openIterator("unknown") - check: - res.isErr() - res.error() == "rocksdb: unknown column family" - test "Empty column family": - let res = db.openIterator(CF_EMPTY) + let res = db.openIterator(emptyCfHandle) check res.isOk() var iter = res.get() defer: @@ -168,7 +165,7 @@ suite "RocksIteratorRef Tests": check not iter.isValid() test "Test status": - let res = db.openIterator(CF_EMPTY) + let res = db.openIterator(emptyCfHandle) check res.isOk() var iter = res.get() defer: @@ -179,7 +176,7 @@ suite "RocksIteratorRef Tests": check iter.status().isOk() test "Test pairs iterator": - let res = db.openIterator(CF_DEFAULT) + let res = db.openIterator(defaultCfHandle) check res.isOk() var iter = res.get() diff --git a/tests/test_sstfilewriter.nim b/tests/test_sstfilewriter.nim index b03343e..f21e197 100644 --- a/tests/test_sstfilewriter.nim +++ b/tests/test_sstfilewriter.nim @@ -29,6 +29,8 @@ suite "SstFileWriterRef Tests": dbPath = mkdtemp() / "data" sstFilePath = mkdtemp() / "sst" db = initReadWriteDb(dbPath, columnFamilyNames = @[CF_DEFAULT, CF_OTHER]) + defaultCfHandle = db.getColFamilyHandle(CF_DEFAULT).get() + otherCfHandle = db.getColFamilyHandle(CF_OTHER).get() teardown: db.close() @@ -66,13 +68,13 @@ suite "SstFileWriterRef Tests": writer.put(key3, val3).isOk() writer.finish().isOk() - db.ingestExternalFile(sstFilePath, CF_OTHER).isOk() - db.keyExists(key1, CF_DEFAULT).get() == false - db.keyExists(key2, CF_DEFAULT).get() == false - db.keyExists(key3, CF_DEFAULT).get() == false - db.get(key1, CF_OTHER).get() == val1 - db.get(key2, CF_OTHER).get() == val2 - db.get(key3, CF_OTHER).get() == val3 + db.ingestExternalFile(sstFilePath, otherCfHandle).isOk() + db.keyExists(key1, defaultCfHandle).get() == false + db.keyExists(key2, defaultCfHandle).get() == false + db.keyExists(key3, defaultCfHandle).get() == false + db.get(key1, otherCfHandle).get() == val1 + db.get(key2, otherCfHandle).get() == val2 + db.get(key3, otherCfHandle).get() == val3 test "Test close": let res = openSstFileWriter(sstFilePath) diff --git a/tests/test_transactiondb.nim b/tests/test_transactiondb.nim index a71c5a5..bddde84 100644 --- a/tests/test_transactiondb.nim +++ b/tests/test_transactiondb.nim @@ -25,8 +25,11 @@ suite "TransactionDbRef Tests": val3 = @[byte(3)] setup: - let dbPath = mkdtemp() / "data" - var db = initTransactionDb(dbPath, columnFamilyNames = @[CF_OTHER]) + let + dbPath = mkdtemp() / "data" + db = initTransactionDb(dbPath, columnFamilyNames = @[CF_OTHER]) + defaultCfHandle = db.getColFamilyHandle(CF_DEFAULT).get() + otherCfHandle = db.getColFamilyHandle(CF_OTHER).get() teardown: db.close() @@ -86,7 +89,7 @@ suite "TransactionDbRef Tests": tx.get(key3).get() == val3 test "Test setting column family in beginTransaction": - var tx = db.beginTransaction(columnFamily = CF_OTHER) + var tx = db.beginTransaction(cfHandle = otherCfHandle) defer: tx.close() check not tx.isClosed() @@ -100,19 +103,19 @@ suite "TransactionDbRef Tests": not tx.isClosed() check: - tx.get(key1, CF_DEFAULT).error() == "" - tx.get(key2, CF_DEFAULT).error() == "" - tx.get(key3, CF_DEFAULT).error() == "" - tx.get(key1, CF_OTHER).get() == val1 - tx.get(key2, CF_OTHER).error() == "" - tx.get(key3, CF_OTHER).get() == val3 + tx.get(key1, defaultCfHandle).error() == "" + tx.get(key2, defaultCfHandle).error() == "" + tx.get(key3, defaultCfHandle).error() == "" + tx.get(key1, otherCfHandle).get() == val1 + tx.get(key2, otherCfHandle).error() == "" + tx.get(key3, otherCfHandle).get() == val3 test "Test rollback and commit with multiple transactions": - var tx1 = db.beginTransaction(columnFamily = CF_DEFAULT) + var tx1 = db.beginTransaction(cfHandle = defaultCfHandle) defer: tx1.close() check not tx1.isClosed() - var tx2 = db.beginTransaction(columnFamily = CF_OTHER) + var tx2 = db.beginTransaction(cfHandle = otherCfHandle) defer: tx2.close() check not tx2.isClosed() @@ -130,41 +133,41 @@ suite "TransactionDbRef Tests": not tx2.isClosed() check: - tx1.get(key1, CF_DEFAULT).get() == val1 - tx1.get(key2, CF_DEFAULT).error() == "" - tx1.get(key3, CF_DEFAULT).get() == val3 - tx1.get(key1, CF_OTHER).error() == "" - tx1.get(key2, CF_OTHER).error() == "" - tx1.get(key3, CF_OTHER).error() == "" - - tx2.get(key1, CF_DEFAULT).error() == "" - tx2.get(key2, CF_DEFAULT).error() == "" - tx2.get(key3, CF_DEFAULT).error() == "" - tx2.get(key1, CF_OTHER).get() == val1 - tx2.get(key2, CF_OTHER).error() == "" - tx2.get(key3, CF_OTHER).get() == val3 + tx1.get(key1, defaultCfHandle).get() == val1 + tx1.get(key2, defaultCfHandle).error() == "" + tx1.get(key3, defaultCfHandle).get() == val3 + tx1.get(key1, otherCfHandle).error() == "" + tx1.get(key2, otherCfHandle).error() == "" + tx1.get(key3, otherCfHandle).error() == "" + + tx2.get(key1, defaultCfHandle).error() == "" + tx2.get(key2, defaultCfHandle).error() == "" + tx2.get(key3, defaultCfHandle).error() == "" + tx2.get(key1, otherCfHandle).get() == val1 + tx2.get(key2, otherCfHandle).error() == "" + tx2.get(key3, otherCfHandle).get() == val3 block: let res = tx1.rollback() check: res.isOk() - tx1.get(key1, CF_DEFAULT).error() == "" - tx1.get(key2, CF_DEFAULT).error() == "" - tx1.get(key3, CF_DEFAULT).error() == "" - tx1.get(key1, CF_OTHER).error() == "" - tx1.get(key2, CF_OTHER).error() == "" - tx1.get(key3, CF_OTHER).error() == "" + tx1.get(key1, defaultCfHandle).error() == "" + tx1.get(key2, defaultCfHandle).error() == "" + tx1.get(key3, defaultCfHandle).error() == "" + tx1.get(key1, otherCfHandle).error() == "" + tx1.get(key2, otherCfHandle).error() == "" + tx1.get(key3, otherCfHandle).error() == "" block: let res = tx2.commit() check: res.isOk() - tx2.get(key1, CF_DEFAULT).error() == "" - tx2.get(key2, CF_DEFAULT).error() == "" - tx2.get(key3, CF_DEFAULT).error() == "" - tx2.get(key1, CF_OTHER).get() == val1 - tx2.get(key2, CF_OTHER).error() == "" - tx2.get(key3, CF_OTHER).get() == val3 + tx2.get(key1, defaultCfHandle).error() == "" + tx2.get(key2, defaultCfHandle).error() == "" + tx2.get(key3, defaultCfHandle).error() == "" + tx2.get(key1, otherCfHandle).get() == val1 + tx2.get(key2, otherCfHandle).error() == "" + tx2.get(key3, otherCfHandle).get() == val3 test "Test close": var tx = db.beginTransaction() diff --git a/tests/test_writebatch.nim b/tests/test_writebatch.nim index 3f36cb9..cc1ec6c 100644 --- a/tests/test_writebatch.nim +++ b/tests/test_writebatch.nim @@ -25,8 +25,11 @@ suite "WriteBatchRef Tests": val3 = @[byte(3)] setup: - let dbPath = mkdtemp() / "data" - var db = initReadWriteDb(dbPath, columnFamilyNames = @[CF_DEFAULT, CF_OTHER]) + let + dbPath = mkdtemp() / "data" + db = initReadWriteDb(dbPath, columnFamilyNames = @[CF_DEFAULT, CF_OTHER]) + defaultCfHandle = db.getColFamilyHandle(CF_DEFAULT).get() + otherCfHandle = db.getColFamilyHandle(CF_OTHER).get() teardown: db.close() @@ -62,57 +65,27 @@ suite "WriteBatchRef Tests": not batch.isClosed() test "Test writing batch to column family": - var batch = db.openWriteBatch() + var batch = db.openWriteBatch(otherCfHandle) defer: batch.close() check not batch.isClosed() check: - batch.put(key1, val1, CF_OTHER).isOk() - batch.put(key2, val2, CF_OTHER).isOk() - batch.put(key3, val3, CF_OTHER).isOk() + batch.put(key1, val1).isOk() + batch.put(key2, val2).isOk() + batch.put(key3, val3).isOk() batch.count() == 3 - batch.delete(key2, CF_OTHER).isOk() - batch.count() == 4 - not batch.isClosed() - - let res = db.write(batch) - check: - res.isOk() - db.get(key1, CF_OTHER).get() == val1 - db.keyExists(key2, CF_OTHER).get() == false - db.get(key3, CF_OTHER).get() == val3 - - batch.clear() - check: - batch.count() == 0 - not batch.isClosed() - - test "Test writing to multiple column families in single batch": - var batch = db.openWriteBatch() - defer: - batch.close() - check not batch.isClosed() - - check: - batch.put(key1, val1).isOk() - batch.put(key1, val1, CF_OTHER).isOk() - batch.put(key2, val2, CF_OTHER).isOk() - batch.put(key3, val3, CF_OTHER).isOk() + batch.delete(key2).isOk() batch.count() == 4 - - batch.delete(key2, CF_OTHER).isOk() - batch.count() == 5 not batch.isClosed() let res = db.write(batch) check: res.isOk() - db.get(key1).get() == val1 - db.get(key1, CF_OTHER).get() == val1 - db.keyExists(key2, CF_OTHER).get() == false - db.get(key3, CF_OTHER).get() == val3 + db.get(key1, otherCfHandle).get() == val1 + db.keyExists(key2, otherCfHandle).get() == false + db.get(key3, otherCfHandle).get() == val3 batch.clear() check: @@ -120,23 +93,25 @@ suite "WriteBatchRef Tests": not batch.isClosed() test "Test writing to multiple column families in multiple batches": - var batch1 = db.openWriteBatch() + var batch1 = db.openWriteBatch(defaultCfHandle) defer: batch1.close() check not batch1.isClosed() - var batch2 = db.openWriteBatch() + var batch2 = db.openWriteBatch(otherCfHandle) defer: batch2.close() check not batch2.isClosed() check: batch1.put(key1, val1).isOk() - batch1.delete(key2, CF_OTHER).isOk() - batch1.put(key3, val3, CF_OTHER).isOk() - batch2.put(key1, val1, CF_OTHER).isOk() - batch2.delete(key1, CF_OTHER).isOk() + batch1.delete(key2).isOk() + batch1.put(key3, val3).isOk() + + batch2.put(key1, val1).isOk() + batch2.delete(key1).isOk() batch2.put(key3, val3).isOk() + batch1.count() == 3 batch2.count() == 3 @@ -148,23 +123,9 @@ suite "WriteBatchRef Tests": db.get(key1).get() == val1 db.keyExists(key2).get() == false db.get(key3).get() == val3 - db.keyExists(key1, CF_OTHER).get() == false - db.keyExists(key2, CF_OTHER).get() == false - db.get(key3, CF_OTHER).get() == val3 - - test "Test unknown column family": - const CF_UNKNOWN = "unknown" - - var batch = db.openWriteBatch() - defer: - batch.close() - check not batch.isClosed() - - let r = batch.put(key1, val1, CF_UNKNOWN) - check r.isErr() and r.error() == "rocksdb: unknown column family" - - let r2 = batch.delete(key1, CF_UNKNOWN) - check r2.isErr() and r2.error() == "rocksdb: unknown column family" + db.keyExists(key1, otherCfHandle).get() == false + db.keyExists(key2, otherCfHandle).get() == false + db.get(key3, otherCfHandle).get() == val3 test "Test write empty batch": var batch = db.openWriteBatch()