diff --git a/rocksdb/backup.nim b/rocksdb/backup.nim index 7c88bc5..e0fb906 100644 --- a/rocksdb/backup.nim +++ b/rocksdb/backup.nim @@ -39,7 +39,8 @@ proc openBackupEngine*( let backupEnginePtr = rocksdb_backup_engine_open( backupOpts.cPtr, path.cstring, cast[cstringArray](errors.addr) ) - bailOnErrors(errors, backupOpts = backupOpts) + + bailOnErrorsAutoCloseOpts(errors, backupOpts = backupOpts) let engine = BackupEngineRef(cPtr: backupEnginePtr, path: path, backupOpts: backupOpts) diff --git a/rocksdb/columnfamily/cfopts.nim b/rocksdb/columnfamily/cfopts.nim index ae7e237..82304af 100644 --- a/rocksdb/columnfamily/cfopts.nim +++ b/rocksdb/columnfamily/cfopts.nim @@ -13,8 +13,10 @@ import ../lib/librocksdb, ../options/tableopts type SlicetransformPtr* = ptr rocksdb_slicetransform_t + SlicetransformRef* = ref object cPtr: SlicetransformPtr + autoClose*: bool # if true then close will be called when the parent is closed ColFamilyOptionsPtr* = ptr rocksdb_options_t @@ -23,6 +25,8 @@ type # type - CF options are a subset of rocksdb_options_t - when in doubt, check: # https://github.com/facebook/rocksdb/blob/b8c9a2576af6a1d0ffcfbb517dfcb7e7037bd460/include/rocksdb/options.h#L66 cPtr: ColFamilyOptionsPtr + sliceTransform: SlicetransformRef + tableOpts: TableOptionsRef autoClose*: bool # if true then close will be called when the database is closed Compression* {.pure.} = enum @@ -36,8 +40,11 @@ type xpressCompression = rocksdb_xpress_compression zstdCompression = rocksdb_zstd_compression -proc createFixedPrefix*(value: int): SlicetransformRef = - SlicetransformRef(cPtr: rocksdb_slicetransform_create_fixed_prefix(value.csize_t)) +proc createFixedPrefix*(value: int, autoClose = false): SlicetransformRef = + SlicetransformRef( + cPtr: rocksdb_slicetransform_create_fixed_prefix(value.csize_t), + autoClose: autoClose, + ) proc isClosed*(s: SlicetransformRef): bool {.inline.} = s.cPtr.isNil() @@ -66,6 +73,11 @@ proc close*(cfOpts: ColFamilyOptionsRef) = rocksdb_options_destroy(cfOpts.cPtr) cfOpts.cPtr = nil + if not cfOpts.sliceTransform.isNil() and cfOpts.sliceTransform.autoClose: + cfOpts.sliceTransform.close() + if not cfOpts.tableOpts.isNil() and cfOpts.tableOpts.autoClose: + cfOpts.tableOpts.close() + template opt(nname, ntyp, ctyp: untyped) = proc `nname=`*(cfOpts: ColFamilyOptionsRef, value: ntyp) = doAssert not cfOpts.isClosed @@ -125,13 +137,21 @@ proc defaultColFamilyOptions*(autoClose = false): ColFamilyOptionsRef = proc `setPrefixExtractor`*(cfOpts: ColFamilyOptionsRef, value: SlicetransformRef) = doAssert not cfOpts.isClosed() + doAssert cfOpts.sliceTransform.isNil() + # don't allow overwriting an existing sliceTransform which could leak memory + rocksdb_options_set_prefix_extractor(cfOpts.cPtr, value.cPtr) + cfOpts.sliceTransform = value proc `blockBasedTableFactory=`*( cfOpts: ColFamilyOptionsRef, tableOpts: TableOptionsRef ) = doAssert not cfOpts.isClosed() + doAssert cfOpts.tableOpts.isNil() + # don't allow overwriting an existing tableOpts which could leak memory + rocksdb_options_set_block_based_table_factory(cfOpts.cPtr, tableOpts.cPtr) + cfOpts.tableOpts = tableOpts # https://github.com/facebook/rocksdb/wiki/MemTable proc setHashSkipListRep*( diff --git a/rocksdb/internal/utils.nim b/rocksdb/internal/utils.nim index 2de5923..2e0896f 100644 --- a/rocksdb/internal/utils.nim +++ b/rocksdb/internal/utils.nim @@ -25,7 +25,7 @@ template autoCloseNonNil*(opts: typed) = if not opts.isNil and opts.autoClose: opts.close() -template bailOnErrors*( +template bailOnErrorsAutoCloseOpts*( errors: cstring, dbOpts: DbOptionsRef = nil, readOpts: ReadOptionsRef = nil, @@ -48,3 +48,9 @@ template bailOnErrors*( let res = err($(errors)) rocksdb_free(errors) return res + +template bailOnErrors*(errors: cstring): auto = + if not errors.isNil: + let res = err($(errors)) + rocksdb_free(errors) + return res diff --git a/rocksdb/options/cache.nim b/rocksdb/options/cache.nim index 9d53b3a..637084b 100644 --- a/rocksdb/options/cache.nim +++ b/rocksdb/options/cache.nim @@ -5,9 +5,10 @@ type CacheRef* = ref object cPtr*: CachePtr + autoClose*: bool # if true then close will be called when the parent type is closed -proc cacheCreateLRU*(size: int): CacheRef = - CacheRef(cPtr: rocksdb_cache_create_lru(size.csize_t)) +proc cacheCreateLRU*(size: int, autoClose = false): CacheRef = + CacheRef(cPtr: rocksdb_cache_create_lru(size.csize_t), autoClose: autoClose) proc close*(cache: CacheRef) = if cache.cPtr != nil: diff --git a/rocksdb/options/dbopts.nim b/rocksdb/options/dbopts.nim index e3f9c0f..ec7eb6f 100644 --- a/rocksdb/options/dbopts.nim +++ b/rocksdb/options/dbopts.nim @@ -18,6 +18,7 @@ type DbOptionsRef* = ref object cPtr: DbOptionsPtr + cache: CacheRef autoClose*: bool # if true then close will be called when the database is closed proc newDbOptions*(autoClose = false): DbOptionsRef = @@ -94,7 +95,11 @@ opt avoidUnnecessaryBlockingIo, bool, uint8 proc `rowCache=`*(dbOpts: DbOptionsRef, cache: CacheRef) = doAssert not dbOpts.isClosed() + doAssert dbOpts.cache.isNil() + # don't allow overwriting an existing cache which could leak memory + rocksdb_options_set_row_cache(dbOpts.cPtr, cache.cPtr) + dbOpts.cache = cache proc defaultDbOptions*(autoClose = false): DbOptionsRef = let opts: DbOptionsRef = newDbOptions(autoClose) @@ -118,3 +123,6 @@ proc close*(dbOpts: DbOptionsRef) = if not dbOpts.isClosed(): rocksdb_options_destroy(dbOpts.cPtr) dbOpts.cPtr = nil + + if not dbOpts.cache.isNil() and dbOpts.cache.autoClose: + dbOpts.cache.close() diff --git a/rocksdb/options/tableopts.nim b/rocksdb/options/tableopts.nim index e8bdba8..4d1071a 100644 --- a/rocksdb/options/tableopts.nim +++ b/rocksdb/options/tableopts.nim @@ -6,11 +6,15 @@ type TableOptionsRef* = ref object cPtr*: TableOptionsPtr + cache: CacheRef + filterPolicy: FilterPolicyRef + autoClose*: bool # if true then close will be called when the parent type is closed FilterPolicyPtr* = ptr rocksdb_filterpolicy_t FilterPolicyRef* = ref object cPtr*: FilterPolicyPtr + autoClose*: bool # if true then close will be called when the parent type is closed IndexType* {.pure.} = enum binarySearch = rocksdb_block_based_table_index_type_binary_search @@ -22,14 +26,17 @@ type binarySearchAndHash = rocksdb_block_based_table_data_block_index_type_binary_search_and_hash -proc createRibbon*(bitsPerKey: float): FilterPolicyRef = - FilterPolicyRef(cPtr: rocksdb_filterpolicy_create_ribbon(bitsPerKey)) +proc createRibbon*(bitsPerKey: float, autoClose = false): FilterPolicyRef = + FilterPolicyRef( + cPtr: rocksdb_filterpolicy_create_ribbon(bitsPerKey), autoClose: autoClose + ) proc createRibbonHybrid*( - bitsPerKey: float, bloomBeforeLevel: int = 0 + bitsPerKey: float, bloomBeforeLevel: int = 0, autoClose = false ): FilterPolicyRef = FilterPolicyRef( - cPtr: rocksdb_filterpolicy_create_ribbon_hybrid(bitsPerKey, bloomBeforeLevel.cint) + cPtr: rocksdb_filterpolicy_create_ribbon_hybrid(bitsPerKey, bloomBeforeLevel.cint), + autoClose: autoClose, ) proc isClosed*(policy: FilterPolicyRef): bool = @@ -40,8 +47,8 @@ proc close*(policy: FilterPolicyRef) = rocksdb_filterpolicy_destroy(policy.cPtr) policy.cPtr = nil -proc createTableOptions*(): TableOptionsRef = - TableOptionsRef(cPtr: rocksdb_block_based_options_create()) +proc createTableOptions*(autoClose = false): TableOptionsRef = + TableOptionsRef(cPtr: rocksdb_block_based_options_create(), autoClose: autoClose) proc isClosed*(opts: TableOptionsRef): bool = isNil(opts.cPtr) @@ -51,6 +58,11 @@ proc close*(opts: TableOptionsRef) = rocksdb_block_based_options_destroy(opts.cPtr) opts.cPtr = nil + if not opts.cache.isNil() and opts.cache.autoClose: + opts.cache.close() + if not opts.filterPolicy.isNil() and opts.filterPolicy.autoClose: + opts.filterPolicy.close() + template opt(nname, ntyp, ctyp: untyped) = proc `nname=`*(opts: TableOptionsRef, value: ntyp) = doAssert not opts.isClosed @@ -76,15 +88,26 @@ opt wholeKeyFiltering, bool, uint8 opt formatVersion, int, cint proc `blockCache=`*(opts: TableOptionsRef, cache: CacheRef) = + doAssert not opts.isClosed() + doAssert opts.cache.isNil() + # don't allow overwriting an existing cache which could leak memory + rocksdb_block_based_options_set_block_cache(opts.cPtr, cache.cPtr) + opts.cache = cache proc `filterPolicy=`*(opts: TableOptionsRef, policy: FilterPolicyRef) = + doAssert not opts.isClosed() + doAssert opts.filterPolicy.isNil() + # don't allow overwriting an existing policy which could leak memory + rocksdb_block_based_options_set_filter_policy(opts.cPtr, policy.cPtr) + opts.filterPolicy = policy -proc defaultTableOptions*(): TableOptionsRef = +proc defaultTableOptions*(autoClose = false): TableOptionsRef = # https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning#other-general-options - let opts = createTableOptions() + let opts = createTableOptions(autoClose) opts.blockSize = 16 * 1024 opts.cacheIndexAndFilterBlocks = true opts.pinL0FilterAndIndexBlocksInCache = true + opts diff --git a/rocksdb/rocksdb.nim b/rocksdb/rocksdb.nim index a5443d7..9086eab 100644 --- a/rocksdb/rocksdb.nim +++ b/rocksdb/rocksdb.nim @@ -77,7 +77,7 @@ proc listColumnFamilies*(path: string): RocksDBResult[seq[string]] = cfList = rocksdb_list_column_families( dbOpts.cPtr, path.cstring, addr cfLen, cast[cstringArray](errors.addr) ) - bailOnErrors(errors, dbOpts) + bailOnErrorsAutoCloseOpts(errors, dbOpts) if cfList.isNil or cfLen == 0: return ok(newSeqOfCap[string](0)) @@ -127,7 +127,7 @@ proc openRocksDb*( cfHandles[0].addr, cast[cstringArray](errors.addr), ) - bailOnErrors(errors, dbOpts, readOpts, writeOpts, cfDescriptors = cfs) + bailOnErrorsAutoCloseOpts(errors, dbOpts, readOpts, writeOpts, cfDescriptors = cfs) let cfTable = newColFamilyTable(cfNames.mapIt($it), cfHandles) @@ -181,7 +181,7 @@ proc openRocksDbReadOnly*( errorIfWalFileExists.uint8, cast[cstringArray](errors.addr), ) - bailOnErrors(errors, dbOpts, readOpts, cfDescriptors = cfs) + bailOnErrorsAutoCloseOpts(errors, dbOpts, readOpts, cfDescriptors = cfs) let cfTable = newColFamilyTable(cfNames.mapIt($it), cfHandles) diff --git a/rocksdb/sstfilewriter.nim b/rocksdb/sstfilewriter.nim index 4160850..5350bd5 100644 --- a/rocksdb/sstfilewriter.nim +++ b/rocksdb/sstfilewriter.nim @@ -44,7 +44,7 @@ proc openSstFileWriter*( rocksdb_sstfilewriter_open( writer.cPtr, filePath.cstring, cast[cstringArray](errors.addr) ) - bailOnErrors(errors, dbOpts) + bailOnErrorsAutoCloseOpts(errors, dbOpts) ok(writer) diff --git a/rocksdb/transactiondb.nim b/rocksdb/transactiondb.nim index f449781..fb0f0ff 100644 --- a/rocksdb/transactiondb.nim +++ b/rocksdb/transactiondb.nim @@ -72,7 +72,7 @@ proc openTransactionDb*( cfHandles[0].addr, cast[cstringArray](errors.addr), ) - bailOnErrors(errors, dbOpts, txDbOpts = txDbOpts, cfDescriptors = cfs) + bailOnErrorsAutoCloseOpts(errors, dbOpts, txDbOpts = txDbOpts, cfDescriptors = cfs) let cfTable = newColFamilyTable(cfNames.mapIt($it), cfHandles)