Metadata rework

.githooks/pre-commit

@@ -5,6 +5,7 @@ export PROFILEDIR=debug
 make fmt-ci &&
 	make build &&
 	make stratisd-tools &&
+	make build-test-extras &&
 	make build-min &&
 	make build-no-ipc &&
 	make test &&

.github/workflows/fedora.yml

@@ -45,6 +45,9 @@ jobs:
           - task: PROFILEDIR=debug make -f Makefile build
             toolchain: 1.79.0  # CURRENT DEVELOPMENT RUST TOOLCHAIN
             components: cargo
+          - task: PROFILEDIR=debug make -f Makefile build-test-extras
+            toolchain: 1.72.0  # CURRENT DEVELOPMENT RUST TOOLCHAIN
+            components: cargo
           - task: PROFILEDIR=debug make -f Makefile build-min
             toolchain: 1.79.0  # CURRENT DEVELOPMENT RUST TOOLCHAIN
             components: cargo
@@ -66,14 +69,12 @@ jobs:
           - task: make -f Makefile test
             toolchain: 1.79.0  # CURRENT DEVELOPMENT RUST TOOLCHAIN
             components: cargo
-          - task: >-
-              TANG_URL=localhost
-              make -f Makefile test-clevis-loop-should-fail
-            toolchain: 1.79.0  # CURRENT DEVELOPMENT RUST TOOLCHAIN
-            components: cargo
           - task: make -f Makefile build
             toolchain: 1.79.0  # CURRENT DEVELOPMENT RUST TOOLCHAIN
             components: cargo
+          - task: make -f Makefile build-test-extras
+            toolchain: 1.72.0  # CURRENT DEVELOPMENT RUST TOOLCHAIN
+            components: cargo
           - task: make -f Makefile build-min
             toolchain: 1.79.0  # CURRENT DEVELOPMENT RUST TOOLCHAIN
             components: cargo
@@ -151,3 +152,51 @@ jobs:
         run: udevadm control --reload
       - name: Test ${{ matrix.task }} on ${{ matrix.toolchain }} toolchain
         run: ${{ matrix.task }}
+
+  # TESTS WITH UDEV
+  checks_with_tang_should_fail:
+    strategy:
+      matrix:
+        include:
+          - task: >-
+              TANG_URL=localhost
+              make -f Makefile test-clevis-loop-should-fail
+            toolchain: 1.78.0  # CURRENT DEVELOPMENT RUST TOOLCHAIN
+            components: cargo
+    runs-on: ubuntu-22.04
+    container:
+      image: fedora:40  # CURRENT DEVELOPMENT ENVIRONMENT
+      options: --privileged -v /dev:/dev -v /run/udev:/run/udev -v /usr/lib/udev:/usr/lib/udev --ipc=host
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install dependencies for Fedora
+        run: >
+          dnf install -y
+          asciidoc
+          clang
+          clevis
+          cryptsetup-devel
+          curl
+          dbus-devel
+          glibc-static
+          device-mapper-devel
+          device-mapper-persistent-data
+          libblkid-devel
+          make
+          ncurses
+          sudo
+          systemd-devel
+          systemd-udev
+          xfsprogs
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          components: ${{ matrix.components }}
+          toolchain: ${{ matrix.toolchain }}
+      - name: Build stratisd
+        run: PROFILEDIR=debug make -f Makefile build-all
+      - name: Install stratisd
+        run: PROFILEDIR=debug make -f Makefile install
+      - name: Reload udev
+        run: udevadm control --reload
+      - name: Test ${{ matrix.task }} on ${{ matrix.toolchain }} toolchain
+        run: ${{ matrix.task }}

.github/workflows/main.yml

@@ -243,10 +243,12 @@ jobs:
     strategy:
       matrix:
         include:
-          - rev: HEAD
+          - rev: origin/fix-start-pool
             options: --post-test-check=verify-filesystem-symlinks --post-test-check=verify-pool-metadata --highest-revision-number=7
+            repo: https://github.com/jbaublitz/testing
           - rev: v3.6.2
             options: --highest-revision-number=6
+            repo: https://github.com/stratis-storage/testing
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
@@ -288,7 +290,7 @@ jobs:
       - name: Reload udev
         run: sudo udevadm control --reload
       - name: Clone testing repo
-        run: git clone https://github.com/stratis-storage/testing.git
+        run: git clone ${{ matrix.repo }}
       - name: Checkout revision ${{ matrix.rev }}
         run: git checkout -b new-branch ${{ matrix.rev }}
         working-directory: ./testing

Cargo.toml

@@ -71,6 +71,10 @@ required-features = ["udev_scripts"]
 name = "stratis-utils"
 required-features = ["engine"]
 
+[[bin]]
+name = "stratis-legacy-pool"
+required-features = ["test_extras"]
+
 [dependencies.async-trait]
 version = "0.1.51"
 optional = true
@@ -205,7 +209,7 @@ version = "0.10.1"
 optional = true
 
 [dependencies.stratisd_proc_macros]
-version = "0.2.0"
+version = "0.2.1"
 optional = true
 path = "./stratisd_proc_macros"
 
@@ -285,6 +289,7 @@ extras = ["pretty-hex"]
 min = ["termios"]
 systemd_compat = ["bindgen"]
 udev_scripts = ["data-encoding"]
+test_extras = ["engine"]
 
 [package.metadata.vendor-filter]
 platforms = ["*-unknown-linux-gnu"]

Makefile

@@ -51,6 +51,7 @@ MIN_FEATURES = --no-default-features --features engine,min
 NO_IPC_FEATURES = --no-default-features --features engine
 SYSTEMD_FEATURES = --no-default-features --features engine,min,systemd_compat
 EXTRAS_FEATURES =  --no-default-features --features engine,extras,min
+TEST_EXTRAS_FEATURES = --no-default-features --features test_extras
 UDEV_FEATURES = --no-default-features --features udev_scripts
 UTILS_FEATURES = --no-default-features --features engine,systemd_compat
 
@@ -293,6 +294,14 @@ stratisd-tools:
 	cargo ${BUILD} ${RELEASE_FLAG} \
 	--bin=stratisd-tools ${EXTRAS_FEATURES} ${TARGET_ARGS}
 
+## Build the test extras
+build-test-extras:
+	PKG_CONFIG_ALLOW_CROSS=1 \
+	RUSTFLAGS="${DENY}" \
+	cargo build ${RELEASE_FLAG} \
+	--bin=stratis-legacy-pool ${TEST_EXTRAS_FEATURES} ${TARGET_ARGS}
+
+## Build the stratis-dumpmetadata program
 ## Build stratis-min for early userspace
 stratis-min:
 	PKG_CONFIG_ALLOW_CROSS=1 \
@@ -514,8 +523,12 @@ clippy-utils:
 clippy-no-ipc:
 	RUSTFLAGS="${DENY}" cargo clippy ${CLIPPY_OPTS} ${NO_IPC_FEATURES} -- ${CLIPPY_DENY} ${CLIPPY_PEDANTIC} ${CLIPPY_PEDANTIC_USELESS}
 
+## Run clippy on no-ipc-build
+clippy-test-extras:
+	RUSTFLAGS="${DENY}" cargo clippy ${CLIPPY_OPTS} ${TEST_EXTRAS_FEATURES} -- ${CLIPPY_DENY} ${CLIPPY_PEDANTIC} ${CLIPPY_PEDANTIC_USELESS}
+
 ## Run clippy on the current source tree
-clippy: clippy-macros clippy-min clippy-udev-utils clippy-no-ipc clippy-utils
+clippy: clippy-macros clippy-min clippy-udev-utils clippy-no-ipc clippy-utils clippy-test-extras
 	RUSTFLAGS="${DENY}" cargo clippy ${CLIPPY_OPTS} -- ${CLIPPY_DENY} ${CLIPPY_PEDANTIC} ${CLIPPY_PEDANTIC_USELESS}
 
 ## Lint Python parts of the source code
@@ -530,6 +543,7 @@ pylint:
 	build-all-man
 	build-all-rust
 	build-min
+	build-test-extras
 	build-udev-utils
 	build-stratis-base32-decode
 	build-stratis-str-cmp
@@ -542,6 +556,7 @@ pylint:
 	clippy-macros
 	clippy-min
 	clippy-no-ipc
+	clippy-test-extras
 	clippy-udev-utils
 	docs-ci
 	docs-rust

plans/all.fmf

@@ -1,5 +1,6 @@
 summary: top level management
 
+enabled: true
 adjust:
   when: plan == cockpit
   enabled: false
@@ -11,28 +12,54 @@ prepare:
   - name: Install packages
     how: install
     package:
-     - tang
+     - cargo
+     - clang
+     - cryptsetup-devel
+     - curl
+     - dbus-devel
+     - device-mapper-devel
+     - libblkid-devel
+     - make
+     - ncurses
+     - rust
      - systemd
      - swtpm
      - swtpm-tools
      - tpm2-tools
+     - systemd-devel
+     - tang
   - name: Start TPM2 emulation
     how: shell
     script: mkdir /var/tmp/swtpm; swtpm_setup --tpm-state /var/tmp/swtpm --tpm2; swtpm chardev --vtpm-proxy --tpmstate dir=/var/tmp/swtpm --tpm2 &> /var/log/swtpm &
   - name: Start tang server
     how: shell
     script: systemctl enable tangd.socket --now
-  - name: Reload udev
-    how: shell
-    script: udevadm control --reload
   - name: Show test system information
     how: shell
     script: free -m; lsblk -i; lscpu; cat /proc/1/sched
   - name: Record mkfs.xfs version
     how: shell
     script: mkfs.xfs -V
-discover:
-  how: fmf
-execute:
-  how: tmt
-  exit-first: false
+
+/python:
+  prepare:
+    - name: Build and install legacy pool script
+      how: shell
+      script:
+        - PROFILEDIR=debug make build-test-extras
+        - mv target/debug/stratis-legacy-pool /usr/local/bin
+
+  discover:
+    how: fmf
+    filter: "tag:python"
+  execute:
+    how: tmt
+    exit-first: false
+
+/rust:
+  discover:
+    how: fmf
+    filter: "tag:rust"
+  execute:
+    how: tmt
+    exit-first: false

src/bin/stratis-legacy-pool.rs

@@ -0,0 +1,132 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+use std::{env, path::PathBuf};
+
+use clap::{Arg, ArgAction, ArgGroup, Command};
+use serde_json::{json, Map, Value};
+
+use stratisd::{
+    engine::{
+        register_clevis_token, EncryptionInfo, KeyDescription, ProcessedPathInfos, StratPool,
+        CLEVIS_TANG_TRUST_URL,
+    },
+    stratis::StratisResult,
+};
+
+fn stratis_legacy_pool_args() -> Command {
+    Command::new("stratis-legacy-pool")
+        .arg(Arg::new("pool_name").num_args(1).required(true))
+        .arg(
+            Arg::new("blockdevs")
+                .action(ArgAction::Append)
+                .required(true),
+        )
+        .arg(
+            Arg::new("key_desc")
+                .long("key-desc")
+                .num_args(1)
+                .required(false),
+        )
+        .arg(
+            Arg::new("clevis")
+                .long("clevis")
+                .num_args(1)
+                .required(false)
+                .value_parser(["nbde", "tang", "tpm2"])
+                .requires_if("nbde", "tang_args")
+                .requires_if("tang", "tang_args"),
+        )
+        .arg(
+            Arg::new("tang_url")
+                .long("tang-url")
+                .num_args(1)
+                .required_if_eq("clevis", "nbde")
+                .required_if_eq("clevis", "tang"),
+        )
+        .arg(Arg::new("thumbprint").long("thumbprint").num_args(1))
+        .arg(Arg::new("trust_url").long("trust-url").num_args(0))
+        .group(
+            ArgGroup::new("tang_args")
+                .arg("thumbprint")
+                .arg("trust_url"),
+        )
+}
+
+type ParseReturn = StratisResult<(
+    String,
+    Vec<PathBuf>,
+    Option<KeyDescription>,
+    Option<(String, Value)>,
+)>;
+
+fn parse_args() -> ParseReturn {
+    let args = env::args().collect::<Vec<_>>();
+    let parser = stratis_legacy_pool_args();
+    let matches = parser.get_matches_from(args);
+
+    let pool_name = matches
+        .get_one::<String>("pool_name")
+        .expect("required")
+        .clone();
+    let blockdevs = matches
+        .get_many::<String>("blockdevs")
+        .expect("required")
+        .map(PathBuf::from)
+        .collect::<Vec<_>>();
+    let key_desc = match matches.get_one::<String>("key_desc") {
+        Some(kd) => Some(KeyDescription::try_from(kd)?),
+        None => None,
+    };
+    let pin = matches.get_one::<String>("clevis");
+    let clevis_info = match pin.map(|s| s.as_str()) {
+        Some("nbde" | "tang") => {
+            let mut json = Map::new();
+            json.insert(
+                "url".to_string(),
+                Value::from(
+                    matches
+                        .get_one::<String>("tang_url")
+                        .expect("Required")
+                        .clone(),
+                ),
+            );
+            if matches.get_flag("trust_url") {
+                json.insert(CLEVIS_TANG_TRUST_URL.to_string(), Value::from(true));
+            } else if let Some(thp) = matches.get_one::<String>("thumbprint") {
+                json.insert("thp".to_string(), Value::from(thp.clone()));
+            }
+            pin.map(|p| (p.to_string(), Value::from(json)))
+        }
+        Some("tpm2") => Some(("tpm2".to_string(), json!({}))),
+        Some(_) => unreachable!("Validated by parser"),
+        None => None,
+    };
+
+    Ok((pool_name, blockdevs, key_desc, clevis_info))
+}
+
+fn main() -> StratisResult<()> {
+    env_logger::init();
+
+    let (name, devices, key_desc, clevis_info) = parse_args()?;
+    let unowned = ProcessedPathInfos::try_from(
+        devices
+            .iter()
+            .map(|p| p.as_path())
+            .collect::<Vec<_>>()
+            .as_slice(),
+    )?
+    .unpack()
+    .1;
+    let encryption_info = match (key_desc, clevis_info) {
+        (Some(kd), Some(ci)) => Some(EncryptionInfo::Both(kd, ci)),
+        (Some(kd), _) => Some(EncryptionInfo::KeyDesc(kd)),
+        (_, Some(ci)) => Some(EncryptionInfo::ClevisInfo(ci)),
+        (_, _) => None,
+    };
+    register_clevis_token()?;
+    StratPool::initialize(name.as_str(), unowned, encryption_info.as_ref())?;
+    Ok(())
+}