lib/kv: Fix j_kv_get_callback for client-side backends #209
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # JULEA - Flexible storage framework | |
| # Copyright (C) 2019-2023 Michael Kuhn | |
| # | |
| # This program is free software: you can redistribute it and/or modify | |
| # it under the terms of the GNU Lesser General Public License as published by | |
| # the Free Software Foundation, either version 3 of the License, or | |
| # (at your option) any later version. | |
| # | |
| # This program is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| # GNU Lesser General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU Lesser General Public License | |
| # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| name: CI | |
| on: [push, pull_request] | |
| jobs: | |
| style: | |
| name: Style | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: Check format | |
| run: | | |
| ./scripts/format.sh | |
| - name: Check shell scripts | |
| run: | | |
| ./scripts/check.sh | |
| dependencies: | |
| name: Dependencies | |
| runs-on: ${{ matrix.os.dist }} | |
| timeout-minutes: 180 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - dist: ubuntu-22.04 | |
| compiler: gcc | |
| compiler_version: 11.4.0 | |
| - dist: ubuntu-22.04 | |
| compiler: clang | |
| compiler_version: 14.0.0 | |
| - dist: ubuntu-20.04 | |
| compiler: gcc | |
| compiler_version: 9.4.0 | |
| - dist: ubuntu-20.04 | |
| compiler: clang | |
| compiler_version: 11.0.0 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: Cache dependencies | |
| id: cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: dependencies | |
| key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} | |
| - name: Install dependencies | |
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | |
| env: | |
| JULEA_SPACK_COMPILER: ${{ matrix.os.compiler }}@${{ matrix.os.compiler_version }} | |
| run: | | |
| ./scripts/install-dependencies.sh | |
| dependencies-containers: | |
| name: Dependencies (Containers) | |
| runs-on: ubuntu-22.04 | |
| container: ${{ matrix.os.registry }}/${{ matrix.os.image }} | |
| timeout-minutes: 180 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - image: ubuntu:22.04 | |
| registry: docker.io/library | |
| compiler: gcc | |
| compiler_version: 11.4.0 | |
| - image: ubuntu:22.04 | |
| registry: docker.io/library | |
| compiler: clang | |
| compiler_version: 14.0.0 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: Install packages | |
| if: ${{ matrix.os.image == 'ubuntu:22.04' }} | |
| run: | | |
| apt update | |
| apt --yes --no-install-recommends install build-essential zstd | |
| apt --yes --no-install-recommends install ca-certificates git gnupg patchelf python3 unzip | |
| if test "${{ matrix.os.compiler }}" = 'clang' | |
| then | |
| apt --yes --no-install-recommends install clang | |
| fi | |
| - name: Cache dependencies | |
| id: cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: /dependencies | |
| key: ${{ matrix.os.image }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} | |
| - name: Install dependencies | |
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | |
| env: | |
| JULEA_SPACK_COMPILER: ${{ matrix.os.compiler }}@${{ matrix.os.compiler_version }} | |
| JULEA_SPACK_DIR: /dependencies | |
| run: | | |
| ./scripts/install-dependencies.sh | |
| build: | |
| name: Build | |
| needs: dependencies | |
| runs-on: ${{ matrix.os.dist }} | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - dist: ubuntu-22.04 | |
| compiler: gcc | |
| compiler_version: 11.4.0 | |
| - dist: ubuntu-22.04 | |
| compiler: clang | |
| compiler_version: 14.0.0 | |
| - dist: ubuntu-20.04 | |
| compiler: gcc | |
| compiler_version: 9.4.0 | |
| - dist: ubuntu-20.04 | |
| compiler: clang | |
| compiler_version: 11.0.0 | |
| dependencies: [system, spack] | |
| exclude: | |
| # libfabric is too old on Ubuntu 20.04 | |
| - os: | |
| dist: ubuntu-20.04 | |
| dependencies: system | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: Clean up system packages | |
| # We need to remove glib-network, otherwise libproxy might cause crashes. | |
| run: | | |
| sudo apt --yes purge glib-networking | |
| sudo apt --yes --purge autoremove | |
| sudo aa-remove-unknown | |
| - name: Install dependencies | |
| if: ${{ matrix.dependencies == 'system' }} | |
| run: | | |
| sudo apt update || true | |
| sudo apt --yes --no-install-recommends install ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev | |
| if test "${{ matrix.os.dist }}" = 'ubuntu-22.04' | |
| then | |
| sudo apt --yes --no-install-recommends install meson | |
| else | |
| sudo apt --yes --no-install-recommends install python3 python3-pip python3-setuptools python3-wheel | |
| sudo pip3 install meson | |
| fi | |
| - name: Cache dependencies | |
| id: cache | |
| if: ${{ matrix.dependencies == 'spack' }} | |
| uses: actions/cache@v3 | |
| with: | |
| path: dependencies | |
| key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} | |
| - name: Check dependencies | |
| if: ${{ matrix.dependencies == 'spack' && steps.cache.outputs.cache-hit != 'true' }} | |
| run: | | |
| exit 1 | |
| - name: Configure | |
| env: | |
| CC: ${{ matrix.os.compiler }} | |
| run: | | |
| . scripts/environment.sh | |
| meson setup --prefix="${GITHUB_WORKSPACE}/julea-install" --buildtype=release --werror bld | |
| - name: Build | |
| run: | | |
| . scripts/environment.sh | |
| ninja -C bld | |
| - name: Install | |
| run: | | |
| . scripts/environment.sh | |
| ninja -C bld install | |
| tests: | |
| name: Tests | |
| needs: dependencies | |
| runs-on: ${{ matrix.os.dist }} | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - dist: ubuntu-22.04 | |
| compiler: gcc | |
| compiler_version: 11.4.0 | |
| - dist: ubuntu-22.04 | |
| compiler: clang | |
| compiler_version: 14.0.0 | |
| - dist: ubuntu-20.04 | |
| compiler: gcc | |
| compiler_version: 9.4.0 | |
| - dist: ubuntu-20.04 | |
| compiler: clang | |
| compiler_version: 11.0.0 | |
| dependencies: [system, spack] | |
| julea: | |
| # Default | |
| - object: posix | |
| kv: lmdb | |
| db: sqlite | |
| # Object backends | |
| - object: gio | |
| kv: lmdb | |
| db: sqlite | |
| # KV backends | |
| - object: posix | |
| kv: mongodb | |
| db: sqlite | |
| - object: posix | |
| kv: leveldb | |
| db: sqlite | |
| - object: posix | |
| kv: rocksdb | |
| db: sqlite | |
| - object: posix | |
| kv: sqlite | |
| db: sqlite | |
| # DB backends | |
| - object: posix | |
| kv: lmdb | |
| db: mysql | |
| db-server: mysql | |
| - object: posix | |
| kv: lmdb | |
| db: mysql | |
| db-server: mariadb | |
| exclude: | |
| # MongoDB is not available on Ubuntu 22.04 | |
| - os: | |
| dist: ubuntu-22.04 | |
| julea: | |
| kv: mongodb | |
| # libfabric is too old on Ubuntu 20.04 | |
| - os: | |
| dist: ubuntu-20.04 | |
| dependencies: system | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: Clean up system packages | |
| # We need to remove glib-network, otherwise libproxy might cause crashes. | |
| run: | | |
| sudo apt --yes purge glib-networking | |
| sudo apt --yes --purge autoremove | |
| sudo aa-remove-unknown | |
| - name: Install dependencies | |
| if: ${{ matrix.dependencies == 'system' }} | |
| run: | | |
| sudo apt update || true | |
| sudo apt --yes --no-install-recommends install ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev | |
| if test "${{ matrix.os.dist }}" = 'ubuntu-22.04' | |
| then | |
| sudo apt --yes --no-install-recommends install meson | |
| else | |
| sudo apt --yes --no-install-recommends install python3 python3-pip python3-setuptools python3-wheel | |
| sudo pip3 install meson | |
| fi | |
| - name: Cache dependencies | |
| id: cache | |
| if: ${{ matrix.dependencies == 'spack' }} | |
| uses: actions/cache@v3 | |
| with: | |
| path: dependencies | |
| key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} | |
| - name: Check dependencies | |
| if: ${{ matrix.dependencies == 'spack' && steps.cache.outputs.cache-hit != 'true' }} | |
| run: | | |
| exit 1 | |
| - name: Set up MongoDB | |
| if: ${{ matrix.julea.kv == 'mongodb' }} | |
| run: | | |
| sudo apt update || true | |
| sudo apt --yes --no-install-recommends install mongodb-server | |
| sudo systemctl start mongodb.service | |
| - name: Set up MySQL | |
| if: ${{ matrix.julea.db == 'mysql' && matrix.julea.db-server == 'mysql' }} | |
| run: | | |
| sudo systemctl start mysql.service | |
| mysql --user=root --password=root --execute="CREATE DATABASE juleadb;" | |
| mysql --user=root --password=root --execute="CREATE USER 'julea'@'localhost' IDENTIFIED WITH mysql_native_password BY 'aeluj';" | |
| mysql --user=root --password=root --execute="GRANT ALL ON juleadb.* TO 'julea'@'localhost';" | |
| - name: Set up MariaDB | |
| if: ${{ matrix.julea.db == 'mysql' && matrix.julea.db-server == 'mariadb' }} | |
| # FIXME https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 | |
| # Ubuntu 20.04 and 22.04 have MySQL 8.0 | |
| run: | | |
| sudo apt --yes purge mysql-client mysql-client-8.0 mysql-server mysql-server-8.0 | |
| sudo apt --yes --purge autoremove | |
| sudo aa-remove-unknown | |
| sudo rm --recursive --force /var/lib/mysql | |
| sudo apt update || true | |
| sudo apt --yes --no-install-recommends install mariadb-server | |
| sudo systemctl start mariadb.service | |
| sudo mariadb --execute="CREATE DATABASE juleadb;" | |
| sudo mariadb --execute="CREATE USER 'julea'@'localhost' IDENTIFIED BY 'aeluj';" | |
| sudo mariadb --execute="GRANT ALL ON juleadb.* TO 'julea'@'localhost';" | |
| - name: Configure | |
| env: | |
| CC: ${{ matrix.os.compiler }} | |
| run: | | |
| . scripts/environment.sh | |
| if test "${{ matrix.os.compiler }}" = 'clang' | |
| then | |
| meson setup -Db_sanitize=address,undefined -Db_lundef=false bld | |
| else | |
| meson setup -Db_sanitize=address,undefined bld | |
| fi | |
| - name: Build | |
| run: | | |
| . scripts/environment.sh | |
| ninja -C bld | |
| - name: Create configuration | |
| # FIXME We need to use 127.0.0.1 because localhost tries to use the socket (/tmp/mysql.sock), which does not exist. | |
| run: | | |
| . scripts/environment.sh | |
| JULEA_KV_COMPONENT='server' | |
| if test "${{ matrix.julea.kv }}" = 'mongodb'; then JULEA_KV_COMPONENT='client'; fi | |
| JULEA_KV_PATH="/tmp/julea/kv/${{ matrix.julea.kv }}" | |
| if test "${{ matrix.julea.kv }}" = 'mongodb'; then JULEA_KV_PATH='127.0.0.1:juleadb'; fi | |
| JULEA_DB_COMPONENT='server' | |
| if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_COMPONENT='client'; fi | |
| JULEA_DB_PATH="/tmp/julea/db/${{ matrix.julea.db }}" | |
| if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_PATH='127.0.0.1:juleadb:julea:aeluj'; fi | |
| julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend="${{ matrix.julea.object }}" --object-component=server --object-path="/tmp/julea/object/${{ matrix.julea.object }}" --kv-backend="${{ matrix.julea.kv }}" --kv-component="${JULEA_KV_COMPONENT}" --kv-path="${JULEA_KV_PATH}" --db-backend="${{ matrix.julea.db }}" --db-component="${JULEA_DB_COMPONENT}" --db-path="${JULEA_DB_PATH}" | |
| - name: Tests | |
| run: | | |
| . scripts/environment.sh | |
| ./scripts/setup.sh start | |
| ./scripts/test.sh | |
| sleep 10 | |
| ./scripts/test.sh | |
| ./scripts/setup.sh stop | |
| - name: HDF5 Tests | |
| env: | |
| LSAN_OPTIONS: exitcode=0 | |
| run: | | |
| . scripts/environment.sh | |
| ./scripts/setup.sh start | |
| export HDF5_VOL_CONNECTOR=julea-kv | |
| ./scripts/test.sh -r /hdf5 | |
| sleep 10 | |
| ./scripts/test.sh -r /hdf5 | |
| sleep 10 | |
| export HDF5_VOL_CONNECTOR=julea-db | |
| ./scripts/test.sh -r /hdf5 | |
| sleep 10 | |
| ./scripts/test.sh -r /hdf5 | |
| ./scripts/setup.sh stop | |
| doxygen: | |
| name: Doxygen | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: Install dependencies | |
| run: | | |
| sudo apt update || true | |
| sudo apt --yes --no-install-recommends install doxygen | |
| - name: Generate documentation | |
| run: | | |
| doxygen | |
| - name: Upload doxygen | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: doxygen | |
| path: html | |
| coverage: | |
| # FIXME Reduce redundancies | |
| # FIXME Increase coverage (currently, only one configuration runs) | |
| name: Coverage | |
| needs: dependencies | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: Cache dependencies | |
| id: cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: dependencies | |
| key: ubuntu-22.04-gcc-11.4.0-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} | |
| - name: Check dependencies | |
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | |
| run: | | |
| exit 1 | |
| - name: Configure | |
| env: | |
| CC: gcc | |
| run: | | |
| . scripts/environment.sh | |
| meson setup -Db_coverage=true -Db_sanitize=address,undefined bld | |
| - name: Build | |
| run: | | |
| . scripts/environment.sh | |
| ninja -C bld | |
| - name: Create configuration | |
| run: | | |
| . scripts/environment.sh | |
| julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend=posix --object-component=server --object-path=/tmp/julea/object/posix --kv-backend=lmdb --kv-component=server --kv-path=/tmp/julea/kv/lmdb --db-backend=sqlite --db-component=server --db-path=/tmp/julea/db/sqlite | |
| - name: Tests | |
| env: | |
| HDF5_VOL_CONNECTOR: julea-db | |
| LSAN_OPTIONS: exitcode=0 | |
| run: | | |
| . scripts/environment.sh | |
| ./scripts/setup.sh start | |
| ./scripts/test.sh | |
| ./scripts/setup.sh stop | |
| - name: Generate coverage | |
| run: | | |
| . scripts/environment.sh | |
| ninja -C bld coverage | |
| - name: Upload coverage | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage | |
| path: bld/meson-logs/coveragereport | |
| publish: | |
| name: Publish | |
| needs: [doxygen, coverage] | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| contents: write | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: gh-pages | |
| path: pages | |
| - name: Download doxygen | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: doxygen | |
| path: doxygen | |
| - name: Download coverage | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: coverage | |
| path: coverage | |
| - name: Prepare | |
| run: | | |
| rsync --archive --human-readable --verbose --delete --exclude=.git doxygen/. pages | |
| mkdir pages/coverage | |
| rsync --archive --human-readable --verbose --delete --exclude=.git coverage/. pages/coverage | |
| touch pages/.nojekyll | |
| - name: Commit | |
| run: | | |
| cd pages | |
| git config user.name 'github-actions[bot]' | |
| git config user.email 'github-actions[bot]@users.noreply.github.com' | |
| git add --all . | |
| git commit --amend --reset-author --message='Pages for ${{ github.sha }}' | |
| git push --force |