From df3c610d22bb7a9910432ae308d0efcd956582f6 Mon Sep 17 00:00:00 2001 From: web3-developer <51288821+web3-developer@users.noreply.github.com> Date: Sun, 7 Jul 2024 22:29:01 +0800 Subject: [PATCH] Add support for optimistic transaction db. --- rocksdb/optimistictxdb.nim | 131 ++++++++++++ rocksdb/transactiondb.nim | 4 +- rocksdb/transactions/otxopts.nim | 49 +++++ rocksdb/transactions/transaction.nim | 6 +- tests/test_all.nim | 2 + tests/test_helper.nim | 16 +- tests/test_optimistictxdb.nim | 291 +++++++++++++++++++++++++++ tests/transactions/test_otxopts.nim | 36 ++++ tests/transactions/test_txdbopts.nim | 6 +- tests/transactions/test_txopts.nim | 6 +- 10 files changed, 536 insertions(+), 11 deletions(-) create mode 100644 rocksdb/optimistictxdb.nim create mode 100644 rocksdb/transactions/otxopts.nim create mode 100644 tests/test_optimistictxdb.nim create mode 100644 tests/transactions/test_otxopts.nim diff --git a/rocksdb/optimistictxdb.nim b/rocksdb/optimistictxdb.nim new file mode 100644 index 0000000..e54ff46 --- /dev/null +++ b/rocksdb/optimistictxdb.nim @@ -0,0 +1,131 @@ +# Nim-RocksDB +# Copyright 2024 Status Research & Development GmbH +# Licensed under either of +# +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +# +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +## A `OptimisticTxDbRef` can be used to open a connection to the RocksDB database +## with support for transactional operations against multiple column families. +## To create a new transaction call `beginTransaction` which will return a +## `TransactionRef`. To commit or rollback the transaction call `commit` or +## `rollback` on the `TransactionRef` type after applying changes to the transaction. + +{.push raises: [].} + +import + std/[sequtils, locks], + ./lib/librocksdb, + ./options/[dbopts, readopts, writeopts], + ./transactions/[transaction, otxopts], + ./columnfamily/[cfopts, cfdescriptor, cfhandle], + ./internal/[cftable, utils], + ./rocksresult + +export dbopts, cfdescriptor, readopts, writeopts, otxopts, transaction, rocksresult + +type + OptimisticTxDbPtr* = ptr rocksdb_optimistictransactiondb_t + + OptimisticTxDbRef* = ref object + lock: Lock + cPtr: OptimisticTxDbPtr + path: string + dbOpts: DbOptionsRef + cfDescriptors: seq[ColFamilyDescriptor] + defaultCfHandle: ColFamilyHandleRef + cfTable: ColFamilyTableRef + +proc openOptimisticTxDb*( + path: string, + dbOpts = defaultDbOptions(autoClose = true), + columnFamilies: openArray[ColFamilyDescriptor] = [], +): RocksDBResult[OptimisticTxDbRef] = + ## Open a `OptimisticTxDbRef` with the given options and column families. + ## If no column families are provided the default column family will be used. + ## If no options are provided the default options will be used. + ## These default options will be closed when the database is closed. + ## If any options are provided, they will need to be closed manually. + + var cfs = columnFamilies.toSeq() + if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()): + cfs.add(defaultColFamilyDescriptor(autoClose = true)) + + var + cfNames = cfs.mapIt(it.name().cstring) + cfOpts = cfs.mapIt(it.options.cPtr) + cfHandles = newSeq[ColFamilyHandlePtr](cfs.len) + errors: cstring + + let txDbPtr = rocksdb_optimistictransactiondb_open_column_families( + dbOpts.cPtr, + path.cstring, + cfNames.len().cint, + cast[cstringArray](cfNames[0].addr), + cfOpts[0].addr, + cfHandles[0].addr, + cast[cstringArray](errors.addr), + ) + bailOnErrorsWithCleanup(errors): + autoCloseNonNil(dbOpts) + autoCloseAll(cfs) + + let + cfTable = newColFamilyTable(cfNames.mapIt($it), cfHandles) + db = OptimisticTxDbRef( + lock: createLock(), + cPtr: txDbPtr, + path: path, + dbOpts: dbOpts, + cfDescriptors: cfs, + defaultCfHandle: cfTable.get(DEFAULT_COLUMN_FAMILY_NAME), + cfTable: cfTable, + ) + ok(db) + +proc getColFamilyHandle*( + db: OptimisticTxDbRef, name: string +): RocksDBResult[ColFamilyHandleRef] = + let cfHandle = db.cfTable.get(name) + if cfHandle.isNil(): + err("rocksdb: unknown column family") + else: + ok(cfHandle) + +proc isClosed*(db: OptimisticTxDbRef): bool {.inline.} = + ## Returns `true` if the `OptimisticTxDbRef` has been closed. + db.cPtr.isNil() + +proc beginTransaction*( + db: OptimisticTxDbRef, + readOpts = defaultReadOptions(autoClose = true), + writeOpts = defaultWriteOptions(autoClose = true), + otxOpts = defaultOptimisticTxOptions(autoClose = true), + cfHandle = db.defaultCfHandle, +): TransactionRef = + ## Begin a new transaction against the database. The transaction will default + ## to using the specified column family. If no column family is specified + ## then the default column family will be used. + doAssert not db.isClosed() + + let txPtr = + rocksdb_optimistictransaction_begin(db.cPtr, writeOpts.cPtr, otxOpts.cPtr, nil) + + newTransaction(txPtr, readOpts, writeOpts, nil, otxOpts, cfHandle) + +proc close*(db: OptimisticTxDbRef) = + ## Close the `OptimisticTxDbRef`. + + withLock(db.lock): + if not db.isClosed(): + # the column families should be closed before the database + db.cfTable.close() + + rocksdb_optimistictransactiondb_close(db.cPtr) + db.cPtr = nil + + # opts should be closed after the database is closed + autoCloseNonNil(db.dbOpts) + autoCloseAll(db.cfDescriptors) diff --git a/rocksdb/transactiondb.nim b/rocksdb/transactiondb.nim index ba591d5..d3be25a 100644 --- a/rocksdb/transactiondb.nim +++ b/rocksdb/transactiondb.nim @@ -114,13 +114,11 @@ proc beginTransaction*( ## Begin a new transaction against the database. The transaction will default ## to using the specified column family. If no column family is specified ## then the default column family will be used. - ## - ## doAssert not db.isClosed() let txPtr = rocksdb_transaction_begin(db.cPtr, writeOpts.cPtr, txOpts.cPtr, nil) - newTransaction(txPtr, readOpts, writeOpts, txOpts, cfHandle) + newTransaction(txPtr, readOpts, writeOpts, txOpts, nil, cfHandle) proc close*(db: TransactionDbRef) = ## Close the `TransactionDbRef`. diff --git a/rocksdb/transactions/otxopts.nim b/rocksdb/transactions/otxopts.nim new file mode 100644 index 0000000..89a60c3 --- /dev/null +++ b/rocksdb/transactions/otxopts.nim @@ -0,0 +1,49 @@ +# Nim-RocksDB +# Copyright 2024 Status Research & Development GmbH +# Licensed under either of +# +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +# +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import ../lib/librocksdb + +type + OptimisticTxOptionsPtr* = ptr rocksdb_optimistictransaction_options_t + + OptimisticTxOptionsRef* = ref object + cPtr: OptimisticTxOptionsPtr + autoClose*: bool # if true then close will be called when the transaction is closed + +proc createOptimisticTxOptions*(autoClose = false): OptimisticTxOptionsRef = + OptimisticTxOptionsRef( + cPtr: rocksdb_optimistictransaction_options_create(), autoClose: autoClose + ) + +proc isClosed*(txOpts: OptimisticTxOptionsRef): bool {.inline.} = + txOpts.cPtr.isNil() + +proc cPtr*(txOpts: OptimisticTxOptionsRef): OptimisticTxOptionsPtr = + doAssert not txOpts.isClosed() + txOpts.cPtr + +template setOpt(nname, ntyp, ctyp: untyped) = + proc `nname=`*(txOpts: OptimisticTxOptionsRef, value: ntyp) = + doAssert not txOpts.isClosed() + `rocksdb_optimistictransaction_options_set nname`(txOpts.cPtr, value.ctyp) + +setOpt setSnapshot, bool, uint8 + +proc defaultOptimisticTxOptions*(autoClose = false): OptimisticTxOptionsRef {.inline.} = + let txOpts = createOptimisticTxOptions(autoClose) + + # TODO: set prefered defaults + txOpts + +proc close*(txOpts: OptimisticTxOptionsRef) = + if not txOpts.isClosed(): + rocksdb_optimistictransaction_options_destroy(txOpts.cPtr) + txOpts.cPtr = nil diff --git a/rocksdb/transactions/transaction.nim b/rocksdb/transactions/transaction.nim index 51cc222..41332e3 100644 --- a/rocksdb/transactions/transaction.nim +++ b/rocksdb/transactions/transaction.nim @@ -24,7 +24,7 @@ import ../options/[readopts, writeopts], ../internal/[cftable, utils], ../rocksresult, - ./txopts + ./[txopts, otxopts] export rocksresult @@ -36,6 +36,7 @@ type readOpts: ReadOptionsRef writeOpts: WriteOptionsRef txOpts: TransactionOptionsRef + otxOpts: OptimisticTxOptionsRef defaultCfHandle: ColFamilyHandleRef proc newTransaction*( @@ -43,6 +44,7 @@ proc newTransaction*( readOpts: ReadOptionsRef, writeOpts: WriteOptionsRef, txOpts: TransactionOptionsRef, + otxOpts: OptimisticTxOptionsRef, defaultCfHandle: ColFamilyHandleRef, ): TransactionRef = TransactionRef( @@ -50,6 +52,7 @@ proc newTransaction*( readOpts: readOpts, writeOpts: writeOpts, txOpts: txOpts, + otxOpts: otxOpts, defaultCfHandle: defaultCfHandle, ) @@ -182,3 +185,4 @@ proc close*(tx: TransactionRef) = autoCloseNonNil(tx.readOpts) autoCloseNonNil(tx.writeOpts) autoCloseNonNil(tx.txOpts) + autoCloseNonNil(tx.otxOpts) diff --git a/tests/test_all.nim b/tests/test_all.nim index 62d2fa1..55ce7c1 100644 --- a/tests/test_all.nim +++ b/tests/test_all.nim @@ -18,10 +18,12 @@ import ./options/test_readopts, ./options/test_tableopts, ./options/test_writeopts, + ./transactions/test_otxopts, ./transactions/test_txdbopts, ./transactions/test_txopts, ./test_backup, ./test_columnfamily, + ./test_optimistictxdb, ./test_rocksdb, ./test_rocksiterator, ./test_sstfilewriter, diff --git a/tests/test_helper.nim b/tests/test_helper.nim index 37d2e1e..a7056a7 100644 --- a/tests/test_helper.nim +++ b/tests/test_helper.nim @@ -9,7 +9,7 @@ {.used.} -import std/sequtils, ../rocksdb/backup, ../rocksdb/rocksdb, ../rocksdb/transactiondb +import std/sequtils, ../rocksdb/[backup, rocksdb, transactiondb, optimistictxdb] proc initReadWriteDb*( path: string, columnFamilyNames: openArray[string] = @[] @@ -57,3 +57,17 @@ proc initTransactionDb*( echo res.error() doAssert res.isOk() res.value() + +proc initOptimisticTxDb*( + path: string, columnFamilyNames: openArray[string] = @[] +): OptimisticTxDbRef = + let res = openOptimisticTxDb( + path, + columnFamilies = columnFamilyNames.mapIt( + initColFamilyDescriptor(it, defaultColFamilyOptions(autoClose = true)) + ), + ) + if res.isErr(): + echo res.error() + doAssert res.isOk() + res.value() diff --git a/tests/test_optimistictxdb.nim b/tests/test_optimistictxdb.nim new file mode 100644 index 0000000..7755704 --- /dev/null +++ b/tests/test_optimistictxdb.nim @@ -0,0 +1,291 @@ +# Nim-RocksDB +# Copyright 2024 Status Research & Development GmbH +# Licensed under either of +# +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +# +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.used.} + +import std/os, tempfile, unittest2, ../rocksdb/optimistictxdb, ./test_helper + +suite "OptimisticTxDbRef Tests": + const + CF_DEFAULT = "default" + CF_OTHER = "other" + + let + key1 = @[byte(1)] + val1 = @[byte(1)] + key2 = @[byte(2)] + val2 = @[byte(2)] + key3 = @[byte(3)] + val3 = @[byte(3)] + + setup: + let + dbPath = mkdtemp() / "data" + db = initOptimisticTxDb(dbPath, columnFamilyNames = @[CF_OTHER]) + defaultCfHandle = db.getColFamilyHandle(CF_DEFAULT).get() + otherCfHandle = db.getColFamilyHandle(CF_OTHER).get() + + teardown: + db.close() + removeDir($dbPath) + + # test multiple transactions + test "Test rollback using default column family": + var tx = db.beginTransaction() + defer: + tx.close() + check not tx.isClosed() + + check: + tx.put(key1, val1).isOk() + tx.put(key2, val2).isOk() + tx.put(key3, val3).isOk() + + tx.delete(key2).isOk() + not tx.isClosed() + + check: + tx.get(key1).get() == val1 + tx.get(key2).error() == "" + tx.get(key3).get() == val3 + + let res = tx.rollback() + check: + res.isOk() + tx.get(key1).error() == "" + tx.get(key2).error() == "" + tx.get(key3).error() == "" + + test "Test commit using default column family": + var tx = db.beginTransaction() + defer: + tx.close() + check not tx.isClosed() + + check: + tx.put(key1, val1).isOk() + tx.put(key2, val2).isOk() + tx.put(key3, val3).isOk() + + tx.delete(key2).isOk() + not tx.isClosed() + + check: + tx.get(key1).get() == val1 + tx.get(key2).error() == "" + tx.get(key3).get() == val3 + + let res = tx.commit() + check: + res.isOk() + tx.get(key1).get() == val1 + tx.get(key2).error() == "" + tx.get(key3).get() == val3 + + test "Test setting column family in beginTransaction": + var tx = db.beginTransaction(cfHandle = otherCfHandle) + defer: + tx.close() + check not tx.isClosed() + + check: + tx.put(key1, val1).isOk() + tx.put(key2, val2).isOk() + tx.put(key3, val3).isOk() + + tx.delete(key2).isOk() + not tx.isClosed() + + check: + tx.get(key1, defaultCfHandle).error() == "" + tx.get(key2, defaultCfHandle).error() == "" + tx.get(key3, defaultCfHandle).error() == "" + tx.get(key1, otherCfHandle).get() == val1 + tx.get(key2, otherCfHandle).error() == "" + tx.get(key3, otherCfHandle).get() == val3 + + test "Test rollback and commit with multiple transactions": + var tx1 = db.beginTransaction(cfHandle = defaultCfHandle) + defer: + tx1.close() + check not tx1.isClosed() + var tx2 = db.beginTransaction(cfHandle = otherCfHandle) + defer: + tx2.close() + check not tx2.isClosed() + + check: + tx1.put(key1, val1).isOk() + tx1.put(key2, val2).isOk() + tx1.put(key3, val3).isOk() + tx1.delete(key2).isOk() + not tx1.isClosed() + tx2.put(key1, val1).isOk() + tx2.put(key2, val2).isOk() + tx2.put(key3, val3).isOk() + tx2.delete(key2).isOk() + not tx2.isClosed() + + check: + tx1.get(key1, defaultCfHandle).get() == val1 + tx1.get(key2, defaultCfHandle).error() == "" + tx1.get(key3, defaultCfHandle).get() == val3 + tx1.get(key1, otherCfHandle).error() == "" + tx1.get(key2, otherCfHandle).error() == "" + tx1.get(key3, otherCfHandle).error() == "" + + tx2.get(key1, defaultCfHandle).error() == "" + tx2.get(key2, defaultCfHandle).error() == "" + tx2.get(key3, defaultCfHandle).error() == "" + tx2.get(key1, otherCfHandle).get() == val1 + tx2.get(key2, otherCfHandle).error() == "" + tx2.get(key3, otherCfHandle).get() == val3 + + block: + let res = tx1.rollback() + check: + res.isOk() + tx1.get(key1, defaultCfHandle).error() == "" + tx1.get(key2, defaultCfHandle).error() == "" + tx1.get(key3, defaultCfHandle).error() == "" + tx1.get(key1, otherCfHandle).error() == "" + tx1.get(key2, otherCfHandle).error() == "" + tx1.get(key3, otherCfHandle).error() == "" + + block: + let res = tx2.commit() + check: + res.isOk() + tx2.get(key1, defaultCfHandle).error() == "" + tx2.get(key2, defaultCfHandle).error() == "" + tx2.get(key3, defaultCfHandle).error() == "" + tx2.get(key1, otherCfHandle).get() == val1 + tx2.get(key2, otherCfHandle).error() == "" + tx2.get(key3, otherCfHandle).get() == val3 + + test "Test close": + var tx = db.beginTransaction() + + check not tx.isClosed() + tx.close() + check tx.isClosed() + tx.close() + check tx.isClosed() + + check not db.isClosed() + db.close() + check db.isClosed() + db.close() + check db.isClosed() + + test "Test close multiple tx": + var tx1 = db.beginTransaction() + var tx2 = db.beginTransaction() + + check not db.isClosed() + check not tx1.isClosed() + tx1.close() + check tx1.isClosed() + tx1.close() + check tx1.isClosed() + + check not db.isClosed() + check not tx2.isClosed() + tx2.close() + check tx2.isClosed() + tx2.close() + check tx2.isClosed() + + test "Test auto close enabled": + let + dbPath = mkdtemp() / "autoclose-enabled" + dbOpts = defaultDbOptions(autoClose = true) + columnFamilies = + @[ + initColFamilyDescriptor(CF_DEFAULT, defaultColFamilyOptions(autoClose = true)) + ] + db = openOptimisticTxDb(dbPath, dbOpts, columnFamilies).get() + + check: + dbOpts.isClosed() == false + columnFamilies[0].isClosed() == false + db.isClosed() == false + + db.close() + + check: + dbOpts.isClosed() == true + columnFamilies[0].isClosed() == true + db.isClosed() == true + + test "Test auto close enabled": + let + dbPath = mkdtemp() / "autoclose-disabled" + dbOpts = defaultDbOptions(autoClose = false) + columnFamilies = + @[ + initColFamilyDescriptor( + CF_DEFAULT, defaultColFamilyOptions(autoClose = false) + ) + ] + db = openOptimisticTxDb(dbPath, dbOpts, columnFamilies).get() + + check: + dbOpts.isClosed() == false + columnFamilies[0].isClosed() == false + db.isClosed() == false + + db.close() + + check: + dbOpts.isClosed() == false + columnFamilies[0].isClosed() == false + db.isClosed() == true + + test "Test auto close tx enabled": + let + readOpts = defaultReadOptions(autoClose = true) + writeOpts = defaultWriteOptions(autoClose = true) + otxOpts = defaultOptimisticTxOptions(autoClose = true) + tx = db.beginTransaction(readOpts, writeOpts, otxOpts) + + check: + readOpts.isClosed() == false + writeOpts.isClosed() == false + otxOpts.isClosed() == false + tx.isClosed() == false + + tx.close() + + check: + readOpts.isClosed() == true + writeOpts.isClosed() == true + otxOpts.isClosed() == true + tx.isClosed() == true + + test "Test auto close tx disabled": + let + readOpts = defaultReadOptions(autoClose = false) + writeOpts = defaultWriteOptions(autoClose = false) + otxOpts = defaultOptimisticTxOptions(autoClose = false) + tx = db.beginTransaction(readOpts, writeOpts, otxOpts) + + check: + readOpts.isClosed() == false + writeOpts.isClosed() == false + otxOpts.isClosed() == false + tx.isClosed() == false + + tx.close() + + check: + readOpts.isClosed() == false + writeOpts.isClosed() == false + otxOpts.isClosed() == false + tx.isClosed() == true diff --git a/tests/transactions/test_otxopts.nim b/tests/transactions/test_otxopts.nim new file mode 100644 index 0000000..7be0902 --- /dev/null +++ b/tests/transactions/test_otxopts.nim @@ -0,0 +1,36 @@ +# Nim-RocksDB +# Copyright 2024 Status Research & Development GmbH +# Licensed under either of +# +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +# +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.used.} + +import unittest2, ../../rocksdb/transactions/otxopts + +suite "OptimisticTxOptionsRef Tests": + test "Test createOptimisticTxOptions": + let txOpts = createOptimisticTxOptions() + + check not txOpts.cPtr.isNil() + + txOpts.close() + + test "Test defaultTransactionOptions": + let txOpts = defaultOptimisticTxOptions() + + check not txOpts.cPtr.isNil() + + txOpts.close() + + test "Test close": + let txOpts = defaultOptimisticTxOptions() + + check not txOpts.isClosed() + txOpts.close() + check txOpts.isClosed() + txOpts.close() + check txOpts.isClosed() diff --git a/tests/transactions/test_txdbopts.nim b/tests/transactions/test_txdbopts.nim index 5619395..b9d56df 100644 --- a/tests/transactions/test_txdbopts.nim +++ b/tests/transactions/test_txdbopts.nim @@ -13,21 +13,21 @@ import unittest2, ../../rocksdb/transactions/txdbopts suite "TransactionDbOptionsRef Tests": test "Test newTransactionDbOptions": - var txDbOpts = createTransactionDbOptions() + let txDbOpts = createTransactionDbOptions() check not txDbOpts.cPtr.isNil() txDbOpts.close() test "Test defaultTransactionDbOptions": - var txDbOpts = defaultTransactionDbOptions() + let txDbOpts = defaultTransactionDbOptions() check not txDbOpts.cPtr.isNil() txDbOpts.close() test "Test close": - var txDbOpts = defaultTransactionDbOptions() + let txDbOpts = defaultTransactionDbOptions() check not txDbOpts.isClosed() txDbOpts.close() diff --git a/tests/transactions/test_txopts.nim b/tests/transactions/test_txopts.nim index 9ea00f7..97cd4de 100644 --- a/tests/transactions/test_txopts.nim +++ b/tests/transactions/test_txopts.nim @@ -13,21 +13,21 @@ import unittest2, ../../rocksdb/transactions/txopts suite "TransactionOptionsRef Tests": test "Test newTransactionOptions": - var txOpts = createTransactionOptions() + let txOpts = createTransactionOptions() check not txOpts.cPtr.isNil() txOpts.close() test "Test defaultTransactionOptions": - var txOpts = defaultTransactionOptions() + let txOpts = defaultTransactionOptions() check not txOpts.cPtr.isNil() txOpts.close() test "Test close": - var txOpts = defaultTransactionOptions() + let txOpts = defaultTransactionOptions() check not txOpts.isClosed() txOpts.close()