0.0.14: DataLoader now support video and streaming

- Added `Hub` for resource management - Updated `DataLoader` to support video and streaming - Updated `CI` - Replaced `println!` with `tracing` for logging
jamjamjon · Sep 16, 2024 · 0adddd3 · 0adddd3
1 parent 826da40
commit 0adddd3
Show file tree

Hide file tree

Showing 82 changed files with 1,729 additions and 573 deletions.
diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml
@@ -1,78 +1,203 @@
-name: Rust
+name: Rust-CI
 
 on:
   push:
-    branches: [ "main", "dev" ]
+    branches: [ "main", "dev", "develop", "x", "xy" , "xyz" ]
   pull_request:
     branches: [ "main" ]
 
+env:
+  CARGO_TERM_COLOR: always
+
+
 jobs:
-  check:
-    name: Check
-    runs-on: ${{ matrix.os }}
+  build-on-linux:
+    name: build / linux / ffmpeg ${{ matrix.ffmpeg_version }}
+    runs-on: ubuntu-latest
+    container: jrottenberg/ffmpeg:${{ matrix.ffmpeg_version }}-ubuntu
+
     strategy:
       matrix:
-        os: [ubuntu-latest, macOS-latest, windows-latest]
-        rust: [stable]
+        ffmpeg_version: ["4.3", "4.4", "5.0", "5.1", "6.0", "6.1", "7.0"]
+      fail-fast: false
+
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: |
+          apt update
+          apt install -y --no-install-recommends clang curl pkg-config
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Build
+        run: cargo build
+
+  build-on-macos:
+    name: build / macos / ffmpeg latest
+    runs-on: macos-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: |
+          brew install ffmpeg pkg-config
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@v1
         with:
-          profile: minimal
-          toolchain: ${{ matrix.rust }}
-          override: true
-      - uses: actions-rs/cargo@v1
+          toolchain: stable
+
+      - name: Build
+        run: cargo build
+
+
+  build-on-windows:
+    name: build / windows / ffmpeg latest
+    runs-on: windows-latest
+
+    env:
+      FFMPEG_DOWNLOAD_URL: https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-full-shared.7z
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: |
+          $VCINSTALLDIR = $(& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath)
+          Add-Content $env:GITHUB_ENV "LIBCLANG_PATH=${VCINSTALLDIR}\VC\Tools\LLVM\x64\bin`n"
+          Invoke-WebRequest "${env:FFMPEG_DOWNLOAD_URL}" -OutFile ffmpeg-release-full-shared.7z
+          7z x ffmpeg-release-full-shared.7z
+          mkdir ffmpeg
+          mv ffmpeg-*/* ffmpeg/
+          Add-Content $env:GITHUB_ENV "FFMPEG_DIR=${pwd}\ffmpeg`n"
+          Add-Content $env:GITHUB_PATH "${pwd}\ffmpeg\bin`n"
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@v1
         with:
-          command: check
-          args: --workspace --examples
+          toolchain: stable
+
+      - name: Build
+        run: cargo build
+
+
+  test-on-linux:
+    name: test / linux / ffmpeg ${{ matrix.ffmpeg_version }}
+    runs-on: ubuntu-latest
+    container: jrottenberg/ffmpeg:${{ matrix.ffmpeg_version }}-ubuntu
 
-  test:
-    name: Test
-    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macOS-latest, windows-latest]
-        rust: [stable]
+        ffmpeg_version: ["4.3", "4.4", "5.0", "5.1", "6.0", "6.1", "7.0"]
+      fail-fast: false
+
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: ${{ matrix.rust }}
-          override: true
-      - uses: actions-rs/cargo@v1
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: |
+          apt update
+          apt install -y --no-install-recommends clang curl pkg-config
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@v1
         with:
-          command: test
-          args: --workspace --examples
+          toolchain: stable
+
+      - name: Run Tests with All Features
+        run: cargo test --all-features
+
+      - name: Run Tests in Release Mode
+        run: cargo test --release
+
+  test-on-macos:
+    name: test / macos / ffmpeg latest 
+    runs-on: macos-latest
 
-  fmt:
-    name: Rustfmt
-    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: |
+          brew install ffmpeg pkg-config
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@v1
         with:
-          profile: minimal
           toolchain: stable
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
+
+      - name: Run Tests with All Features
+        run: cargo test --all-features
+
+      - name: Run Tests in Release Mode
+        run: cargo test --release
+
+  test-on-windows:
+    name: test / windows / ffmpeg latest
+    runs-on: windows-latest
+
+    env:
+      FFMPEG_DOWNLOAD_URL: https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-full-shared.7z
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: |
+          $VCINSTALLDIR = $(& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath)
+          Add-Content $env:GITHUB_ENV "LIBCLANG_PATH=${VCINSTALLDIR}\VC\Tools\LLVM\x64\bin`n"
+          Invoke-WebRequest "${env:FFMPEG_DOWNLOAD_URL}" -OutFile ffmpeg-release-full-shared.7z
+          7z x ffmpeg-release-full-shared.7z
+          mkdir ffmpeg
+          mv ffmpeg-*/* ffmpeg/
+          Add-Content $env:GITHUB_ENV "FFMPEG_DIR=${pwd}\ffmpeg`n"
+          Add-Content $env:GITHUB_PATH "${pwd}\ffmpeg\bin`n"
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@v1
         with:
-          command: fmt
-          args: --all -- --check
+          toolchain: stable
+
+      - name: Run Tests with All Features
+        run: cargo test --all-features
+
+      - name: Run Tests in Release Mode
+        run: cargo test --release
 
-  clippy:
-    name: Clippy
+
+  lints:
     runs-on: ubuntu-latest
+    container: jrottenberg/ffmpeg:6-ubuntu
+
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: |
+          apt update
+          apt install -y --no-install-recommends clang curl pkg-config
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@v1
         with:
-          profile: minimal
           toolchain: stable
-          override: true
-      - run: rustup component add clippy
-      - uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --workspace --tests --examples --all-targets --all-features -- -Dwarnings
+          components: rustfmt, clippy
+
+      - name: Rustfmt
+        run: cargo fmt --all -- --check
+
+      - name: Clippy
+        run: cargo clippy --all --all-targets --all-features -- -D warnings
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "usls"
-version = "0.0.13"
+version = "0.0.14"
 edition = "2021"
 description = "A Rust library integrated with ONNXRuntime, providing a collection of ML models."
 repository = "https://github.com/jamjamjon/usls"
@@ -22,7 +22,7 @@ dirs = { version = "5.0.1" }
 ureq = { version = "2.9.1", default-features = true, features = [
     "socks-proxy",
 ] }
-walkdir = { version = "2.5.0" }
+walkdir = { version = "2.5.0" }  # TODO: remove
 tokenizers = { version = "0.15.2" }
 rayon = "1.10.0"
 indicatif = "0.17.8"
@@ -32,6 +32,13 @@ ab_glyph = "0.2.23"
 geo = "0.28.0"
 prost = "0.12.4"
 fast_image_resize = { version = "4.2.1", features = ["image"]}
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+tempfile = "3.12.0"
+video-rs = { version = "0.9.0", features = ["ndarray"] }
+natord = "1.0.9"
+tracing = "0.1.40"
+tracing-subscriber = "0.3.18"
 
 
 [features]

diff --git a/README.md b/README.md
@@ -98,24 +98,93 @@ You have two options to link the ONNXRuntime library
 
 
 
-## 🎈 Quick Start
+## 🎈 Demo
 
 ```Shell
 cargo run -r --example yolo   # blip, clip, yolop, svtr, db, ...
 ```
 
 ## 🥂 Integrate Into Your Own Project
 
-Add `usls` as a dependency to your project's `Cargo.toml`
-```Shell
-cargo add usls
-```
+- #### Add `usls` as a dependency to your project's `Cargo.toml`
+    ```Shell
+    cargo add usls
+    ```
+
+    Or use a specific commit:
+    ```Toml
+    [dependencies]
+    usls = { git = "https://github.com/jamjamjon/usls", rev = "commit-sha" }
+    ```
+    
+- #### Follow the pipeline
+    - Build model with the provided `models` and `Options`
+    - Load images, video and stream with `DataLoader`
+    - Do inference
+    - Annotate inference results with `Annotator`
+    - Retrieve inference results from `Vec<Y>`
+           
+      ```rust
+        use usls::{models::YOLO, Annotator, DataLoader, Nms, Options, Vision, YOLOTask, YOLOVersion};
+    
+        fn main() -> anyhow::Result<()> {
+            // Build model with Options
+            let options = Options::new()
+                .with_trt(0)
+                .with_model("yolo/v8-m-dyn.onnx")?
+                .with_yolo_version(YOLOVersion::V8) // YOLOVersion: V5, V6, V7, V8, V9, V10, RTDETR
+                .with_yolo_task(YOLOTask::Detect) // YOLOTask: Classify, Detect, Pose, Segment, Obb
+                .with_i00((1, 2, 4).into())
+                .with_i02((0, 640, 640).into())
+                .with_i03((0, 640, 640).into())
+                .with_confs(&[0.2]);
+            let mut model = YOLO::new(options)?;
+        
+            // Build DataLoader to load image(s), video, stream
+            let dl = DataLoader::new(
+                // "./assets/bus.jpg", // local image
+                // "images/bus.jpg",  // remote image
+                // "../images-folder",  // local images (from folder)
+                // "../demo.mp4",  // local video
+                // "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4",  // remote video
+                "rtsp://admin:[email protected]:554/h264/ch1/",  // stream
+            )?
+            .with_batch(2)  // iterate with batch_size = 2
+            .build()?;
+        
+            // Build annotator
+            let annotator = Annotator::new()
+                .with_bboxes_thickness(4)
+                .with_saveout("YOLO-DataLoader");
+        
+            // Run and annotate results
+            for (xs, _) in dl {
+                let ys = model.forward(&xs, false)?;
+                annotator.annotate(&xs, &ys);
+      
+                // Retrieve inference results
+                for y in ys {
+                    // bboxes
+                    if let Some(bboxes) = y.bboxes() {
+                        for bbox in bboxes {
+                            println!(
+                                "Bbox: {}, {}, {}, {}, {}, {}",
+                                bbox.xmin(),
+                                bbox.ymin(),
+                                bbox.xmax(),
+                                bbox.ymax(),
+                                bbox.confidence(),
+                                bbox.id(),
+                            );
+                        }
+                    }
+                }
+            }
+        
+            Ok(())
+        }
+      ```
 
-Or use a specific commit:
-```Toml
-[dependencies]
-usls = { git = "https://github.com/jamjamjon/usls", rev = "commit-sha" }
-```
 
 ## 📌 License
 This project is licensed under [LICENSE](LICENSE).
diff --git a/assets/2.jpg b/assets/2.jpg
diff --git a/assets/car.jpg b/assets/car.jpg
diff --git a/assets/db.png b/assets/db.png
diff --git a/assets/dog.jpg b/assets/dog.jpg
diff --git a/assets/dota.png b/assets/dota.png
diff --git a/assets/kids.jpg b/assets/kids.jpg
diff --git a/assets/liuyifei.png b/assets/liuyifei.png
diff --git a/assets/nini.png b/assets/nini.png
diff --git a/assets/paul-george.jpg b/assets/paul-george.jpg
diff --git a/assets/truck.jpg b/assets/truck.jpg