diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 126c8c3..676ca28 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,59 +66,6 @@ jobs: with: submodules: true - - name: Install build dependencies (Linux i386) - if: runner.os == 'Linux' && matrix.target.cpu == 'i386' - run: | - sudo dpkg --add-architecture i386 - sudo apt-fast update -qq - sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \ - --no-install-recommends -yq gcc-multilib g++-multilib \ - libz-dev:i386 libbz2-dev:i386 libssl-dev:i386 liblz4-dev:i386 - mkdir -p external/bin - cat << EOF > external/bin/gcc - #!/bin/bash - exec $(which gcc) -m32 "\$@" - EOF - cat << EOF > external/bin/g++ - #!/bin/bash - exec $(which g++) -m32 "\$@" - EOF - chmod 755 external/bin/gcc external/bin/g++ - echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH - - - name: Restore rocksdb from cache (Linux + macOS) - if: runner.os != 'Windows' - id: rocksdb-cache - uses: actions/cache@v4 - with: - path: rocks-db-cache-${{ matrix.target.cpu }} - key: 'rocksdb-v1-${{ matrix.target.os }}-${{ matrix.target.cpu }}' - - - name: Build and install rocksdb (Linux i386 + macOS) - # no librocksdb-dev:i386 - if: (runner.os == 'Linux' && matrix.target.cpu == 'i386') || runner.os == 'Macos' - run: | - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_rocksdb.sh - bash build_rocksdb.sh rocks-db-cache-${{ matrix.target.cpu }} - - - name: Install rocksdb (Linux amd64) - # mysterious illegal instruction error if we build our own librocksdb - if: runner.os == 'Linux' && matrix.target.cpu == 'amd64' - run: | - sudo apt-get -q update - sudo apt-get install -y librocksdb-dev - - - name: MSYS2 (Windows i386) - if: runner.os == 'Windows' && matrix.target.cpu == 'i386' - uses: msys2/setup-msys2@v2 - with: - path-type: inherit - msystem: MINGW32 - install: >- - base-devel - git - mingw-w64-i686-toolchain - - name: MSYS2 (Windows amd64) if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' uses: msys2/setup-msys2@v2 @@ -129,28 +76,6 @@ jobs: git mingw-w64-x86_64-toolchain - - name: Install DLL dependencies (Windows) - if: > - runner.os == 'Windows' - run: | - if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then - ROCKSDBSUB=x64 - else - ROCKSDBSUB=x86 - fi - DLLPATH="external/dlls-${{ matrix.target.cpu }}" - mkdir -p ${DLLPATH} - - # ROCKSDB - ./scripts/build_dlls_windows.sh - cp ./build/librocksdb.dll "${DLLPATH}/librocksdb.dll" - - - name: Path to cached dependencies (Windows) - if: > - runner.os == 'Windows' - run: | - echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH - - name: Derive environment variables run: | if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then @@ -196,7 +121,7 @@ jobs: nim --version nimble --version - nimble install -y --depsOnly + nimble install -y nimble test # static linking is not supported on windows diff --git a/rocksdb.nimble b/rocksdb.nimble index bd266dc..684ecd6 100644 --- a/rocksdb.nimble +++ b/rocksdb.nimble @@ -22,9 +22,26 @@ task clean, "Remove temporary files": exec "make -C vendor/rocksdb clean" task test, "Run tests": - exec "nim c -r --threads:on tests/test_all.nim" + let runTests = "nim c -r --threads:on tests/test_all.nim" + when defined(linux): + exec "export LD_LIBRARY_PATH=build; " & runTests + when defined(macosx): + exec "export DYLD_LIBRARY_PATH=build; " & runTests + when defined(windows): + exec runTests task test_static, "Run tests after static linking dependencies": - when not defined(windows): - exec "scripts/build_static_deps.sh" + when defined(windows): + echo "Static linking is not supported on windows" + quit(1) + + exec "scripts/build_static_deps.sh" exec "nim c -d:rocksdb_static_linking -r --threads:on tests/test_all.nim" + +before install: + when defined(linux): + exec "scripts/build_shared_deps_linux.sh" + when defined(macosx): + exec "scripts/build_shared_deps_osx.sh" + when defined(windows): + exec ".\\scripts\\build_dlls_windows.bat" diff --git a/rocksdb/lib/librocksdb.nim b/rocksdb/lib/librocksdb.nim index b2bcf8b..89bd919 100644 --- a/rocksdb/lib/librocksdb.nim +++ b/rocksdb/lib/librocksdb.nim @@ -24,10 +24,6 @@ {.push raises: [].} -proc shouldUseNativeLinking(): bool {.compileTime.} = - when defined(linux): - return true - type rocksdb_t* = object rocksdb_backup_engine_t* = object @@ -108,10 +104,6 @@ when defined(rocksdb_static_linking): when defined(windows): {.passl: "-lshlwapi -lrpcrt4".} else: - when shouldUseNativeLinking(): - {.pragma: importrocks, importc, cdecl.} - {.passl: "-lrocksdb".} - else: - {.pragma: importrocks, importc, cdecl, dynlib: librocksdb.} + {.pragma: importrocks, importc, cdecl, dynlib: librocksdb.} include ./rocksdb_gen.nim diff --git a/rocksdb/options/tableopts.nim b/rocksdb/options/tableopts.nim index a980a28..69bf799 100644 --- a/rocksdb/options/tableopts.nim +++ b/rocksdb/options/tableopts.nim @@ -26,11 +26,11 @@ type binarySearchAndHash = rocksdb_block_based_table_data_block_index_type_binary_search_and_hash -proc createRibbon*(bitsPerKey: float, autoClose = false): FilterPolicyRef = +proc createRibbon*(bitsPerKey: float): FilterPolicyRef = FilterPolicyRef(cPtr: rocksdb_filterpolicy_create_ribbon(bitsPerKey)) proc createRibbonHybrid*( - bitsPerKey: float, bloomBeforeLevel: int = 0, autoClose = false + bitsPerKey: float, bloomBeforeLevel: int = 0 ): FilterPolicyRef = FilterPolicyRef( cPtr: rocksdb_filterpolicy_create_ribbon_hybrid(bitsPerKey, bloomBeforeLevel.cint) diff --git a/scripts/build_shared_deps_linux.sh b/scripts/build_shared_deps_linux.sh new file mode 100755 index 0000000..bd43840 --- /dev/null +++ b/scripts/build_shared_deps_linux.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Nim-RocksDB +# Copyright 2024 Status Research & Development GmbH +# Licensed under either of +# +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +# +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")"/.. + +REPO_DIR="${PWD}" +BUILD_DEST="${REPO_DIR}/build/" + + +git submodule update --init + +${REPO_DIR}/vendor/vcpkg/bootstrap-vcpkg.sh -disableMetrics + +${REPO_DIR}/vendor/vcpkg/vcpkg install rocksdb[lz4,zstd]:x64-linux-rocksdb --recurse --overlay-triplets=${REPO_DIR}/triplets + +mkdir -p "${BUILD_DEST}" + +cp "${REPO_DIR}/vendor/vcpkg/installed/x64-linux-rocksdb/lib/liblz4.so" "${BUILD_DEST}/" +cp "${REPO_DIR}/vendor/vcpkg/installed/x64-linux-rocksdb/lib/libzstd.so" "${BUILD_DEST}/" +cp "${REPO_DIR}/vendor/vcpkg/installed/x64-linux-rocksdb/lib/librocksdb.so" "${BUILD_DEST}/" diff --git a/scripts/build_shared_deps_osx.sh b/scripts/build_shared_deps_osx.sh new file mode 100755 index 0000000..3c7eb7f --- /dev/null +++ b/scripts/build_shared_deps_osx.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Nim-RocksDB +# Copyright 2024 Status Research & Development GmbH +# Licensed under either of +# +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +# +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")"/.. + +REPO_DIR="${PWD}" +BUILD_DEST="${REPO_DIR}/build/" + + +git submodule update --init + +${REPO_DIR}/vendor/vcpkg/bootstrap-vcpkg.sh -disableMetrics + +${REPO_DIR}/vendor/vcpkg/vcpkg install rocksdb[lz4,zstd]:x64-osx-rocksdb --recurse --overlay-triplets=${REPO_DIR}/triplets + +mkdir -p "${BUILD_DEST}" + +cp "${REPO_DIR}/vendor/vcpkg/installed/x64-osx-rocksdb/lib/liblz4.dylib" "${BUILD_DEST}/" +cp "${REPO_DIR}/vendor/vcpkg/installed/x64-osx-rocksdb/lib/libzstd.dylib" "${BUILD_DEST}/" +cp "${REPO_DIR}/vendor/vcpkg/installed/x64-osx-rocksdb/lib/librocksdb.dylib" "${BUILD_DEST}/" diff --git a/scripts/generate_wrapper.sh b/scripts/generate_wrapper.sh index a8493ab..2938cf5 100755 --- a/scripts/generate_wrapper.sh +++ b/scripts/generate_wrapper.sh @@ -11,6 +11,8 @@ set -e +nimble install c2nim + cd "$(dirname "${BASH_SOURCE[0]}")"/.. VENDOR_HEADER_FILE="vendor/rocksdb/include/rocksdb/c.h" @@ -71,3 +73,5 @@ sed -i ':a;N;$!ba;s/#ifdef _WIN32\ # generate nim wrapper c2nim ${OUTPUT_HEADER_FILE} --out:"${C2NIM_GENERATED_WRAPPER}" + +nimble format diff --git a/tests/options/test_dbopts.nim b/tests/options/test_dbopts.nim index 66b7921..a29784f 100644 --- a/tests/options/test_dbopts.nim +++ b/tests/options/test_dbopts.nim @@ -20,10 +20,9 @@ suite "DbOptionsRef Tests": dbOpts.maxOpenFiles = 10 dbOpts.createMissingColumnFamilies = false - # TODO rocksdb 5.17.2 used in some tests has no getters for settings exposed! - # check: - # dbOpts.maxOpenFiles == 10 - # not dbOpts.createMissingColumnFamilies + check: + dbOpts.maxOpenFiles == 10 + not dbOpts.createMissingColumnFamilies dbOpts.close() @@ -36,20 +35,19 @@ suite "DbOptionsRef Tests": dbOpts.close() check dbOpts.isClosed() - # This is currently failing in MacOS CI due to older version of RocksDb - # test "Test auto close enabled": - # let - # dbOpts = defaultDbOptions() - # cache = cacheCreateLRU(1000, autoClose = true) + test "Test auto close enabled": + let + dbOpts = defaultDbOptions() + cache = cacheCreateLRU(1000, autoClose = true) - # dbOpts.rowCache = cache + dbOpts.rowCache = cache - # check: - # dbOpts.isClosed() == false - # cache.isClosed() == false + check: + dbOpts.isClosed() == false + cache.isClosed() == false - # dbOpts.close() + dbOpts.close() - # check: - # dbOpts.isClosed() == true - # cache.isClosed() == true + check: + dbOpts.isClosed() == true + cache.isClosed() == true diff --git a/tests/options/test_tableopts.nim b/tests/options/test_tableopts.nim index 076fbf2..d459191 100644 --- a/tests/options/test_tableopts.nim +++ b/tests/options/test_tableopts.nim @@ -36,45 +36,43 @@ suite "TableOptionsRef Tests": check tableOpts.isClosed() test "Test auto close enabled": - # TODO: enable filter policy once creating updated DLL build let tableOpts = defaultTableOptions() cache = cacheCreateLRU(1000, autoClose = true) - # filter = createRibbon(9.9, autoClose = true) + filter = createRibbon(9.9) tableOpts.blockCache = cache - # tableOpts.filterPolicy = filter + tableOpts.filterPolicy = filter check: tableOpts.isClosed() == false cache.isClosed() == false - # filter.isClosed() == true # closed because tableopts takes ownership + filter.isClosed() == true # closed because tableopts takes ownership tableOpts.close() check: tableOpts.isClosed() == true cache.isClosed() == true - # filter.isClosed() == true + filter.isClosed() == true test "Test auto close disabled": - # TODO: enable filter policy once creating updated DLL build let tableOpts = defaultTableOptions() cache = cacheCreateLRU(1000, autoClose = false) - # filter = createRibbon(9.9, autoClose = true) + filter = createRibbon(9.9) tableOpts.blockCache = cache - # tableOpts.filterPolicy = filter + tableOpts.filterPolicy = filter check: tableOpts.isClosed() == false cache.isClosed() == false - # filter.isClosed() == true # closed because tableopts takes ownership + filter.isClosed() == true # closed because tableopts takes ownership tableOpts.close() check: tableOpts.isClosed() == true cache.isClosed() == false - # filter.isClosed() == true + filter.isClosed() == true diff --git a/tests/test_all.nim b/tests/test_all.nim index 3b6287a..0360269 100644 --- a/tests/test_all.nim +++ b/tests/test_all.nim @@ -25,4 +25,5 @@ import ./test_rocksdb, ./test_rocksiterator, ./test_sstfilewriter, - ./test_writebatch + ./test_writebatch, + ./test_transactiondb diff --git a/triplets/x64-linux-rocksdb.cmake b/triplets/x64-linux-rocksdb.cmake new file mode 100644 index 0000000..f2b100b --- /dev/null +++ b/triplets/x64-linux-rocksdb.cmake @@ -0,0 +1,7 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_BUILD_TYPE release) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE dynamic) + +set(VCPKG_CMAKE_SYSTEM_NAME Linux) +set(VCPKG_FIXUP_ELF_RPATH ON) diff --git a/triplets/x64-osx-rocksdb.cmake b/triplets/x64-osx-rocksdb.cmake new file mode 100644 index 0000000..8813618 --- /dev/null +++ b/triplets/x64-osx-rocksdb.cmake @@ -0,0 +1,7 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_BUILD_TYPE release) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE dynamic) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES x86_64) diff --git a/triplets/x86-windows-rocksdb.cmake b/triplets/x86-windows-rocksdb.cmake deleted file mode 100644 index 701a059..0000000 --- a/triplets/x86-windows-rocksdb.cmake +++ /dev/null @@ -1,10 +0,0 @@ -set(VCPKG_TARGET_ARCHITECTURE x86) -set(VCPKG_BUILD_TYPE release) - -if(${PORT} MATCHES "snappy|zlib|lz4|zstd") - set(VCPKG_CRT_LINKAGE static) - set(VCPKG_LIBRARY_LINKAGE static) -else() - set(VCPKG_CRT_LINKAGE static) - set(VCPKG_LIBRARY_LINKAGE dynamic) -endif()