From 656e521b8fcd8d11e119bf01ef928ba22d2187cc Mon Sep 17 00:00:00 2001 From: web3-developer <51288821+web3-developer@users.noreply.github.com> Date: Mon, 19 Feb 2024 14:32:59 +0800 Subject: [PATCH] Commit library rewrite progress. --- .../{descriptor.nim => cfdescriptor.nim} | 13 +- .../columnfamily/{handle.nim => cfhandle.nim} | 26 +-- rocksdb/columnfamily/cfopts.nim | 22 ++- rocksdb/internal/cftable.nim | 41 ++++ rocksdb/internal/utils.nim | 0 rocksdb/options/dbopts.nim | 30 +-- rocksdb/options/readopts.nim | 24 ++- rocksdb/options/writeopts.nim | 24 ++- rocksdb/rocksdb.nim | 180 ++++++++++++------ 9 files changed, 244 insertions(+), 116 deletions(-) rename rocksdb/columnfamily/{descriptor.nim => cfdescriptor.nim} (68%) rename rocksdb/columnfamily/{handle.nim => cfhandle.nim} (64%) create mode 100644 rocksdb/internal/cftable.nim delete mode 100644 rocksdb/internal/utils.nim diff --git a/rocksdb/columnfamily/descriptor.nim b/rocksdb/columnfamily/cfdescriptor.nim similarity index 68% rename from rocksdb/columnfamily/descriptor.nim rename to rocksdb/columnfamily/cfdescriptor.nim index a5c3077..8b4fb7a 100644 --- a/rocksdb/columnfamily/descriptor.nim +++ b/rocksdb/columnfamily/cfdescriptor.nim @@ -12,7 +12,7 @@ import ./cfopts -const DEFAULT_COLUMN_FAMILY* = "default" +const DEFAULT_COLUMN_FAMILY_NAME* = "default" type ColFamilyDescriptor* = object @@ -24,14 +24,17 @@ proc initColFamilyDescriptor*( options = defaultColFamilyOptions()): ColFamilyDescriptor = ColFamilyDescriptor(name: name, options: options) -proc name*(descriptor: ColFamilyDescriptor): string = +template name*(descriptor: ColFamilyDescriptor): string = descriptor.name -proc options*(descriptor: ColFamilyDescriptor): ColFamilyOptionsRef = +template options*(descriptor: ColFamilyDescriptor): ColFamilyOptionsRef = descriptor.options -proc defaultColFamilyDescriptor*(): ColFamilyDescriptor = - initColFamilyDescriptor(DEFAULT_COLUMN_FAMILY) +template isDefault*(descriptor: ColFamilyDescriptor): bool = + descriptor.name() == DEFAULT_COLUMN_FAMILY_NAME + +template defaultColFamilyDescriptor*(): ColFamilyDescriptor = + initColFamilyDescriptor(DEFAULT_COLUMN_FAMILY_NAME) template close*(descriptor: var ColFamilyDescriptor) = descriptor.options.close() diff --git a/rocksdb/columnfamily/handle.nim b/rocksdb/columnfamily/cfhandle.nim similarity index 64% rename from rocksdb/columnfamily/handle.nim rename to rocksdb/columnfamily/cfhandle.nim index 4cd4885..dd754b9 100644 --- a/rocksdb/columnfamily/handle.nim +++ b/rocksdb/columnfamily/cfhandle.nim @@ -11,36 +11,40 @@ import ../lib/librocksdb, - ./descriptor + ./cfdescriptor type ColFamilyHandlePtr* = ptr rocksdb_column_family_handle_t ColFamilyHandleRef* = ref object - handlePtr: ColFamilyHandlePtr + cPtr: ColFamilyHandlePtr -proc newColFamilyHandle*(handlePtr: ColFamilyHandlePtr): ColFamilyHandleRef = - ColFamilyHandleRef(handlePtr: handlePtr) +proc newColFamilyHandle*(cPtr: ColFamilyHandlePtr): ColFamilyHandleRef = + ColFamilyHandleRef(cPtr: cPtr) template isClosed(handle: ColFamilyHandleRef): bool = - handle.handlePtr.isNil() + handle.cPtr.isNil() + +proc cPtr*(handle: ColFamilyHandleRef): ColFamilyHandlePtr = + doAssert not handle.isClosed() + handle.cPtr proc getId*(handle: ColFamilyHandleRef): int = doAssert not handle.isClosed() - rocksdb_column_family_handle_get_id(handle.handlePtr).int + rocksdb_column_family_handle_get_id(handle.cPtr).int proc getName*(handle: ColFamilyHandleRef): string = doAssert not handle.isClosed() var nameLen: csize_t # do we need to use this? - $rocksdb_column_family_handle_get_name(handle.handlePtr, nameLen.addr) + $rocksdb_column_family_handle_get_name(handle.cPtr, nameLen.addr) -proc isDefault*(handle: ColFamilyHandleRef): bool = - handle.getName() == DEFAULT_COLUMN_FAMILY +template isDefault*(handle: ColFamilyHandleRef): bool = + handle.getName() == DEFAULT_COLUMN_FAMILY_NAME # proc getDescriptor*(handle: ColFamilyHandleRef): ColumnFamilyDescriptor = # doAssert not handle.isClosed() proc close*(handle: var ColFamilyHandleRef) = if not handle.isClosed(): - rocksdb_column_family_handle_destroy(handle.handlePtr) - handle.handlePtr = nil \ No newline at end of file + rocksdb_column_family_handle_destroy(handle.cPtr) + handle.cPtr = nil \ No newline at end of file diff --git a/rocksdb/columnfamily/cfopts.nim b/rocksdb/columnfamily/cfopts.nim index 1e14144..cb7b26d 100644 --- a/rocksdb/columnfamily/cfopts.nim +++ b/rocksdb/columnfamily/cfopts.nim @@ -13,20 +13,24 @@ import ../lib/librocksdb type - OptionsPtr = ptr rocksdb_options_t + ColFamilyOptionsPtr* = ptr rocksdb_options_t ColFamilyOptionsRef* = ref object - optionsPtr: OptionsPtr + cPtr: ColFamilyOptionsPtr proc newColFamilyOptions*(): ColFamilyOptionsRef = - ColFamilyOptionsRef(optionsPtr: rocksdb_options_create()) + ColFamilyOptionsRef(cPtr: rocksdb_options_create()) -template isClosed(dbOpts: ColFamilyOptionsRef): bool = - dbOpts.optionsPtr.isNil() +template isClosed(cfOpts: ColFamilyOptionsRef): bool = + cfOpts.cPtr.isNil() + +proc cPtr*(cfOpts: ColFamilyOptionsRef): ColFamilyOptionsPtr = + doAssert not cfOpts.isClosed() + cfOpts.cPtr proc setCreateMissingColumnFamilies*(cfOpts: var ColFamilyOptionsRef, flag: bool) = doAssert not cfOpts.isClosed() - rocksdb_options_set_create_missing_column_families(cfOpts.optionsPtr, flag.uint8) + rocksdb_options_set_create_missing_column_families(cfOpts.cPtr, flag.uint8) proc defaultColFamilyOptions*(): ColFamilyOptionsRef = var opts = newColFamilyOptions() @@ -36,10 +40,10 @@ proc defaultColFamilyOptions*(): ColFamilyOptionsRef = proc getCreateMissingColumnFamilies*(cfOpts: ColFamilyOptionsRef): bool = doAssert not cfOpts.isClosed() - rocksdb_options_get_create_missing_column_families(cfOpts.optionsPtr).bool + rocksdb_options_get_create_missing_column_families(cfOpts.cPtr).bool proc close*(cfOpts: var ColFamilyOptionsRef) = if not cfOpts.isClosed(): - rocksdb_options_destroy(cfOpts.optionsPtr) - cfOpts.optionsPtr = nil + rocksdb_options_destroy(cfOpts.cPtr) + cfOpts.cPtr = nil diff --git a/rocksdb/internal/cftable.nim b/rocksdb/internal/cftable.nim new file mode 100644 index 0000000..3ec4d11 --- /dev/null +++ b/rocksdb/internal/cftable.nim @@ -0,0 +1,41 @@ +# Nim-RocksDB +# Copyright 2024 Status Research & Development GmbH +# Licensed under either of +# +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +# +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import + std/tables, + ../columnfamily/cfhandle + +type + ColFamilyTableRef* = ref object + columnFamilies: TableRef[string, ColFamilyHandleRef] + +proc newColFamilyTableRef*(): ColFamilyTableRef = + ColFamilyTableRef(columnFamilies: newTable[string, ColFamilyHandleRef]()) + +template isClosed(table: ColFamilyTableRef): bool = + table.columnFamilies.isNil() + +proc put*( + table: var ColFamilyTableRef, + name: string, + handle: ColFamilyHandlePtr) = + doAssert not table.isClosed() + table.columnFamilies[name] = newColFamilyHandle(handle) + +proc get*(table: ColFamilyTableRef, name: string): ColFamilyHandleRef = + doAssert not table.isClosed() + table.columnFamilies.getOrDefault(name) + +proc close*(table: var ColFamilyTableRef) = + if not table.isClosed(): + for _, v in table.columnFamilies.mpairs(): + v.close() + table.columnFamilies = nil \ No newline at end of file diff --git a/rocksdb/internal/utils.nim b/rocksdb/internal/utils.nim deleted file mode 100644 index e69de29..0000000 diff --git a/rocksdb/options/dbopts.nim b/rocksdb/options/dbopts.nim index ed9c575..22fae4b 100644 --- a/rocksdb/options/dbopts.nim +++ b/rocksdb/options/dbopts.nim @@ -14,34 +14,38 @@ import ../lib/librocksdb type - OptionsPtr = ptr rocksdb_options_t + DbOptionsPtr* = ptr rocksdb_options_t DbOptionsRef* = ref object - optionsPtr: OptionsPtr + cPtr: DbOptionsPtr proc newDbOptions*(): DbOptionsRef = - DbOptionsRef(optionsPtr: rocksdb_options_create()) + DbOptionsRef(cPtr: rocksdb_options_create()) template isClosed(dbOpts: DbOptionsRef): bool = - dbOpts.optionsPtr.isNil() + dbOpts.cPtr.isNil() + +proc cPtr*(dbOpts: DbOptionsRef): DbOptionsPtr = + doAssert not dbOpts.isClosed() + dbOpts.cPtr proc setIncreaseParallelism*(dbOpts: var DbOptionsRef, totalThreads: int) = doAssert totalThreads > 0 doAssert not dbOpts.isClosed() - rocksdb_options_increase_parallelism(dbOpts.optionsPtr, totalThreads.cint) + rocksdb_options_increase_parallelism(dbOpts.cPtr, totalThreads.cint) proc setCreateIfMissing*(dbOpts: var DbOptionsRef, flag: bool) = doAssert not dbOpts.isClosed() - rocksdb_options_set_create_if_missing(dbOpts.optionsPtr, flag.uint8) + rocksdb_options_set_create_if_missing(dbOpts.cPtr, flag.uint8) proc setMaxOpenFiles*(dbOpts: var DbOptionsRef, maxOpenFiles: int) = doAssert maxOpenFiles > -1 doAssert not dbOpts.isClosed() - rocksdb_options_set_max_open_files(dbOpts.optionsPtr, maxOpenFiles.cint) + rocksdb_options_set_max_open_files(dbOpts.cPtr, maxOpenFiles.cint) proc setCreateMissingColumnFamilies*(dbOpts: var DbOptionsRef, flag: bool) = doAssert not dbOpts.isClosed() - rocksdb_options_set_create_missing_column_families(dbOpts.optionsPtr, flag.uint8) + rocksdb_options_set_create_missing_column_families(dbOpts.cPtr, flag.uint8) proc defaultDbOptions*(): DbOptionsRef = var opts = newDbOptions() @@ -60,18 +64,18 @@ proc defaultDbOptions*(): DbOptionsRef = proc getCreateIfMissing*(dbOpts: DbOptionsRef): bool = doAssert not dbOpts.isClosed() - rocksdb_options_get_create_if_missing(dbOpts.optionsPtr).bool + rocksdb_options_get_create_if_missing(dbOpts.cPtr).bool proc getMaxOpenFiles*(dbOpts: DbOptionsRef): int = doAssert not dbOpts.isClosed() - rocksdb_options_get_max_open_files(dbOpts.optionsPtr).int + rocksdb_options_get_max_open_files(dbOpts.cPtr).int proc getCreateMissingColumnFamilies*(dbOpts: DbOptionsRef): bool = doAssert not dbOpts.isClosed() - rocksdb_options_get_create_missing_column_families(dbOpts.optionsPtr).bool + rocksdb_options_get_create_missing_column_families(dbOpts.cPtr).bool proc close*(dbOpts: var DbOptionsRef) = if not dbOpts.isClosed(): - rocksdb_options_destroy(dbOpts.optionsPtr) - dbOpts.optionsPtr = nil + rocksdb_options_destroy(dbOpts.cPtr) + dbOpts.cPtr = nil diff --git a/rocksdb/options/readopts.nim b/rocksdb/options/readopts.nim index 97fe76c..6d725c8 100644 --- a/rocksdb/options/readopts.nim +++ b/rocksdb/options/readopts.nim @@ -13,25 +13,29 @@ import ../lib/librocksdb type - ReadOptionsPtr = ptr rocksdb_readoptions_t + ReadOptionsPtr* = ptr rocksdb_readoptions_t ReadOptionsRef* = ref object - readOptsPtr: ReadOptionsPtr + cPtr: ReadOptionsPtr proc newReadOptions*(): ReadOptionsRef = - ReadOptionsRef(readOptsPtr: rocksdb_readoptions_create()) - -template defaultReadOptions*(): ReadOptionsRef = - newReadOptions() + ReadOptionsRef(cPtr: rocksdb_readoptions_create()) template isClosed(readOpts: ReadOptionsRef): bool = - readOpts.readOptsPtr.isNil() + readOpts.cPtr.isNil() + +proc cPtr*(readOpts: ReadOptionsRef): ReadOptionsPtr = + doAssert not readOpts.isClosed() + readOpts.cPtr # TODO: Add setters and getters for read options properties. -# Currently we are using the default settings. + +template defaultReadOptions*(): ReadOptionsRef = + newReadOptions() + # TODO: set prefered defaults proc close*(readOpts: var ReadOptionsRef) = if not readOpts.isClosed(): - rocksdb_readoptions_destroy(readOpts.readOptsPtr) - readOpts.readOptsPtr = nil + rocksdb_readoptions_destroy(readOpts.cPtr) + readOpts.cPtr = nil diff --git a/rocksdb/options/writeopts.nim b/rocksdb/options/writeopts.nim index d4ca8cd..61be65b 100644 --- a/rocksdb/options/writeopts.nim +++ b/rocksdb/options/writeopts.nim @@ -13,25 +13,29 @@ import ../lib/librocksdb type - WriteOptionsPtr = ptr rocksdb_writeoptions_t + WriteOptionsPtr* = ptr rocksdb_writeoptions_t WriteOptionsRef* = ref object - writeOptsPtr: WriteOptionsPtr + cPtr: WriteOptionsPtr proc newWriteOptions*(): WriteOptionsRef = - WriteOptionsRef(writeOptsPtr: rocksdb_writeoptions_create()) - -template defaultWriteOptions*(): WriteOptionsRef = - newWriteOptions() + WriteOptionsRef(cPtr: rocksdb_writeoptions_create()) template isClosed(writeOpts: WriteOptionsRef): bool = - writeOpts.writeOptsPtr.isNil() + writeOpts.cPtr.isNil() + +proc cPtr*(writeOpts: WriteOptionsRef): WriteOptionsPtr = + doAssert not writeOpts.isClosed() + writeOpts.cPtr # TODO: Add setters and getters for write options properties. -# Currently we are using the default settings. + +template defaultWriteOptions*(): WriteOptionsRef = + newWriteOptions() + # TODO: set prefered defaults proc close*(writeOpts: var WriteOptionsRef) = if not writeOpts.isClosed(): - rocksdb_writeoptions_destroy(writeOpts.writeOptsPtr) - writeOpts.writeOptsPtr = nil + rocksdb_writeoptions_destroy(writeOpts.cPtr) + writeOpts.cPtr = nil diff --git a/rocksdb/rocksdb.nim b/rocksdb/rocksdb.nim index c741485..e2c64a5 100644 --- a/rocksdb/rocksdb.nim +++ b/rocksdb/rocksdb.nim @@ -10,11 +10,12 @@ {.push raises: [].} import - std/[cpuinfo, tables], + std/[tables, sequtils], results, ./lib/librocksdb, ./options/[dbopts, readopts, writeopts], - ./columnfamily/[cfopts, descriptor, handle] + ./columnfamily/[cfopts, cfdescriptor, cfhandle], + ./internal/cftable export results @@ -22,73 +23,136 @@ type RocksDBResult*[T] = Result[T, string] RocksDbPtr = ptr rocksdb_t - - RocksDbRef* = ref object + RocksDbRef = ref object rocksDbPtr: RocksDbPtr + path: string dbOpts: DbOptionsRef readOpts: ReadOptionsRef writeOpts: WriteOptionsRef - # dbPath: string - # columnFamilyNames: cstringArray - # columnFamilies: TableRef[cstring, ptr rocksdb_column_family_handle_t] + cfTable: ColFamilyTableRef + + RocksDbReadWriteRef* = distinct RocksDbRef + RocksDbReadOnlyRef* = distinct RocksDbRef + + DataProc* = proc(val: openArray[byte]) {.gcsafe, raises: [].} - #DataProc* = proc(val: openArray[byte]) {.gcsafe, raises: [].} +template bailOnErrors(errors: cstring): auto = + if not errors.isNil: + let r = err($(errors)) + rocksdb_free(errors) + return r + +proc openDb[T]( + readOnly: bool, + path: string, + dbOpts: DbOptionsRef, + readOpts: ReadOptionsRef, + writeOpts: WriteOptionsRef, + columnFamilies: openArray[ColFamilyDescriptor], + errorIfWalFileExists: bool): RocksDBResult[T] = + + if columnFamilies.len == 0: + return err("rocksdb: no column families") + + var + cfNames = columnFamilies.mapIt(it.name().cstring) + cfOpts = columnFamilies.mapIt(it.options.cPtr) + columnFamilyHandles = newSeq[ColFamilyHandlePtr](columnFamilies.len) + errors: cstring + + let rocksDbPtr = if readOnly: + rocksdb_open_for_read_only_column_families( + dbOpts.cPtr, + path, + cfNames.len().cint, + cast[cstringArray](cfNames[0].addr), + cfOpts[0].addr, + columnFamilyHandles[0].addr, + errorIfWalFileExists.uint8, + cast[cstringArray](errors.addr)) + else: + rocksdb_open_column_families( + dbOpts.cPtr, + path, + cfNames.len().cint, + cast[cstringArray](cfNames[0].addr), + cfOpts[0].addr, + columnFamilyHandles[0].addr, + cast[cstringArray](errors.addr)) + bailOnErrors(errors) + + var cfTable = newColFamilyTableRef() + for i, cf in columnFamilies: + cfTable.put(cf.name(), columnFamilyHandles[i]) + + let db = RocksDbRef( + rocksDbPtr: rocksDbPtr, + path: path, + dbOpts: dbOpts, + readOpts: readOpts, + writeOpts: writeOpts, + cfTable: cfTable) + ok(db.T) proc openRocksDb*( path: string, dbOpts = defaultDbOptions(), readOpts = defaultReadOptions(), writeOpts = defaultWriteOptions(), - columnFamilies = @[defaultColFamilyDescriptor()]): RocksDBResult[RocksDbRef] = + columnFamilies = @[defaultColFamilyDescriptor()]): RocksDBResult[RocksDbReadWriteRef] = + openDb[RocksDbReadWriteRef]( + false, path, dbOpts, readOpts, writeOpts, columnFamilies, false) + +proc openRocksDbReadOnly*( + path: string, + dbOpts = defaultDbOptions(), + readOpts = defaultReadOptions(), + columnFamilies = @[defaultColFamilyDescriptor()], + errorIfWalFileExists = false): RocksDBResult[RocksDbReadOnlyRef] = + openDb[RocksDbReadOnlyRef]( + true, path, dbOpts, readOpts, nil, columnFamilies, errorIfWalFileExists) + +proc get*( + db: RocksDbReadWriteRef | RocksDbReadOnlyRef, + key: openArray[byte], + columnFamily = "default"): RocksDBResult[seq[byte]] = + discard + +proc get*( + db: RocksDbReadWriteRef | RocksDbReadOnlyRef, + key: openArray[byte], + value: var openArray[byte], + columnFamily = "default"): RocksDBResult[bool] = + discard + +proc get*( + db: RocksDbReadWriteRef | RocksDbReadOnlyRef, + key: openArray[byte], + onData: DataProc, + columnFamily = "default"): RocksDBResult[bool] = + discard + +proc put*( + db: RocksDbReadWriteRef, + key, val: openArray[byte], + columnFamily = "default"): RocksDBResult[void] = + discard + +proc keyExists*( + db: RocksDbReadWriteRef | RocksDbReadOnlyRef, + key: openArray[byte], + columnFamily = "default"): RocksDBResult[bool] = + discard + +proc delete*( + db: RocksDbReadWriteRef, + key: openArray[byte], + columnFamily = "default"): RocksDBResult[void] = + discard +proc backup*( + db: RocksDbReadWriteRef | RocksDbReadOnlyRef): RocksDBResult[void] = discard - # for i in 0..columnFamilyNames.high: - # rocks.options.add(rocksdb_options_create()) - # rocks.readOptions = rocksdb_readoptions_create() - # rocks.writeOptions = rocksdb_writeoptions_create() - # rocks.dbPath = dbPath - # rocks.columnFamilyNames = columnFamilyNames.allocCStringArray - # rocks.columnFamilies = newTable[cstring, ptr rocksdb_column_family_handle_t]() - - # for opts in rocks.options: - # # Optimize RocksDB. This is the easiest way to get RocksDB to perform well: - # rocksdb_options_increase_parallelism(opts, cpus.int32) - # # This requires snappy - disabled because rocksdb is not always compiled with - # # snappy support (for example Fedora 28, certain Ubuntu versions) - # # rocksdb_options_optimize_level_style_compaction(options, 0); - # rocksdb_options_set_create_if_missing(opts, uint8(createIfMissing)) - # # default set to keep all files open (-1), allow setting it to a specific - # # value, e.g. in case the application limit would be reached. - # rocksdb_options_set_max_open_files(opts, maxOpenFiles.cint) - # # Enable creating column families if they do not exist - # rocksdb_options_set_create_missing_column_families(opts, uint8(true)) - - # var - # columnFamilyHandles = newSeq[ptr rocksdb_column_family_handle_t](columnFamilyNames.len) - # errors: cstring - # rocks.db = rocksdb_open_column_families( - # rocks.options[0], - # dbPath, - # columnFamilyNames.len().cint, - # rocks.columnFamilyNames, - # rocks.options[0].addr, - # columnFamilyHandles[0].addr, - # cast[cstringArray](errors.addr)) - # bailOnErrors() - - # for i in 0..