merge

facebookresearch · Jan 10, 2025 · 324a649 · 324a649
2 parents ffd751f + 0a87bd8
commit 324a649
Show file tree

Hide file tree

Showing 431 changed files with 26,061 additions and 11,019 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,3 +1,4 @@
 * @cbalioglu
 src/fairseq2/data/parquet/ @artemru
 tests/integration/parquet/ @artemru
+doc/ @zyaoj
diff --git a/.github/workflows/_build_doc.yaml b/.github/workflows/_build_doc.yaml
@@ -9,31 +9,32 @@ on:
     inputs:
       torch:
         type: string
-        default: '2.3.0'
+        default: '2.5.1'
       py:
         type: string
-        default: '3.11'
+        default: '3.12'
       version_override:
         type: string
         default: ''
 
-env:
-    ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
-
 jobs:
   build:
     name: Build
     runs-on:
       labels: 4-core-ubuntu
     container:
-      image: ghcr.io/facebookresearch/fairseq2-ci-manylinux_x86_64:2-cpu
+      image: ghcr.io/facebookresearch/fairseq2-ci-manylinux_x86_64:3-cpu
     defaults:
       run:
         shell: bash
     steps:
+      - name: Install Git LFS
+        run: |
+          yum install -y git-lfs
       - name: Check-out the repository
         uses: actions/checkout@v3
         with:
+          lfs: true
           submodules: recursive
       - name: Create the Python virtual environment
         run: |
@@ -57,7 +58,7 @@ jobs:
       - name: Configure fairseq2n
         working-directory: native
         run: |
-          cmake -GNinja -B build
+          scl enable gcc-toolset-11 "cmake -GNinja -B build"
       - name: Install fairseq2n
         run: |
           pip install --editable native/python
@@ -72,7 +73,7 @@ jobs:
         run: |
           cp VERSION doc/build/html
       - name: Upload documentation to staging
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: doc
           path: doc/build/html/

diff --git a/.github/workflows/_build_wheel-linux.yaml b/.github/workflows/_build_wheel-linux.yaml
@@ -38,16 +38,13 @@ on:
         type: boolean
         default: false
 
-env:
-    ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
-
 jobs:
   build:
     name: Build
     runs-on:
       labels: 4-core-ubuntu
     container:
-      image: ghcr.io/facebookresearch/fairseq2-ci-manylinux_${{ inputs.arch }}:2-${{ inputs.variant }}
+      image: ghcr.io/facebookresearch/fairseq2-ci-manylinux_${{ inputs.arch }}:3-${{ inputs.variant }}
     defaults:
       run:
         shell: bash
@@ -81,9 +78,9 @@ jobs:
 
           # If the version has already a local label, append the variant.
           if [[ $version == *+* ]]; then
-              tools/set-project-version.sh $version.$VARIANT
+              tools/set-project-version.sh --native-only $version.$VARIANT
           else
-              tools/set-project-version.sh $version+$VARIANT
+              tools/set-project-version.sh --native-only $version+$VARIANT
           fi
       - name: Build fairseq2n
         working-directory: native
@@ -112,18 +109,20 @@ jobs:
             lto=OFF
           fi
 
+          scl enable gcc-toolset-11 - <<EOF
           cmake\
             -GNinja\
             -DCMAKE_BUILD_TYPE=$build_type\
-            -DCMAKE_CUDA_ARCHITECTURES=$cuda_archs\
+            -DCMAKE_CUDA_ARCHITECTURES="$cuda_archs"\
             -DFAIRSEQ2N_PERFORM_LTO=$lto\
             -DFAIRSEQ2N_SANITIZERS="${SANITIZERS/_/;}"\
             -DFAIRSEQ2N_TREAT_WARNINGS_AS_ERRORS=ON\
             -DFAIRSEQ2N_USE_CUDA=$cuda\
             -DFAIRSEQ2N_PYTHON_DEVEL=OFF\
             -B build
+          EOF
 
-          cmake --build build
+          scl enable gcc-toolset-11 "cmake --build build"
       - name: Package fairseq2n
         working-directory: native/python
         run: |
@@ -132,14 +131,14 @@ jobs:
             --no-build-isolation\
             --no-deps\
             --config-settings "--build-option=--plat-name"\
-            --config-settings "--build-option=manylinux2014_${{ inputs.arch }}"\
+            --config-settings "--build-option=manylinux_2_28_${{ inputs.arch }}"\
             --wheel-dir build/wheelhouse\
             .
       - name: Package fairseq2
         run: |
           pip wheel --no-deps --wheel-dir build/wheelhouse .
       - name: Upload wheels and native tests to staging
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.artifacts_name }}-pt${{ inputs.torch }}-py${{ inputs.py }}-linux_${{ inputs.arch }}-${{ inputs.variant }}-${{ inputs.sanitizers }}
           path: |
@@ -154,13 +153,13 @@ jobs:
     runs-on:
       labels: 4-core-ubuntu
     container:
-      image: ghcr.io/facebookresearch/fairseq2-ci-manylinux_${{ inputs.arch }}:2-${{ inputs.variant }}
+      image: ghcr.io/facebookresearch/fairseq2-ci-manylinux_${{ inputs.arch }}:3-${{ inputs.variant }}
     defaults:
       run:
         shell: bash
     steps:
       - name: Download wheels and native tests from staging
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ inputs.artifacts_name }}-pt${{ inputs.torch }}-py${{ inputs.py }}-linux_${{ inputs.arch }}-${{ inputs.variant }}-${{ inputs.sanitizers }}
           path: ~/artifacts/

diff --git a/.github/workflows/_build_wheel-macos.yaml b/.github/workflows/_build_wheel-macos.yaml
@@ -23,9 +23,6 @@ on:
         type: string
         default: ''
 
-env:
-    ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
-
 jobs:
   build:
     name: Build
@@ -84,7 +81,7 @@ jobs:
         run: |
           pip wheel --no-deps --wheel-dir build/wheelhouse .
       - name: Upload wheels and native tests to staging
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.artifacts_name }}-pt${{ inputs.torch }}-py${{ inputs.py }}-macos_${{ inputs.arch }}-cpu-nosan
           path: |
@@ -105,7 +102,7 @@ jobs:
         run: |
           brew install libsndfile python@${{ inputs.py }} || true
       - name: Download wheels and native tests from staging
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ inputs.artifacts_name }}-pt${{ inputs.torch }}-py${{ inputs.py }}-macos_${{ inputs.arch }}-cpu-nosan
           path: ~/artifacts/

diff --git a/.github/workflows/_build_wheels.yaml b/.github/workflows/_build_wheels.yaml
@@ -14,18 +14,15 @@ on:
         type: string
         required: true
 
-env:
-    ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
-
 jobs:
-  build_pt21_wheel-linux:
+  build_pt23_wheel-linux:
     name: Build wheels (pt${{ matrix.torch }}, py${{ matrix.py }}, linux-x86_64, ${{ matrix.variant }}, ${{ matrix.sanitizers }})
     uses: ./.github/workflows/_build_wheel-linux.yaml
     strategy:
       fail-fast: false
       matrix:
-        torch: ['2.1.0', '2.1.1', '2.1.2']
-        py: ['3.8', '3.9', '3.10', '3.11']
+        torch: ['2.3.0', '2.3.1']
+        py: ['3.10', '3.11', '3.12']
         variant: ['cpu', 'cu118', 'cu121']
         sanitizers: ['nosan']
     with:
@@ -35,15 +32,15 @@ jobs:
       sanitizers: ${{ matrix.sanitizers }}
       version_override: ${{ inputs.version_override }}
 
-  build_pt22_wheel-linux:
+  build_pt24_wheel-linux:
     name: Build wheels (pt${{ matrix.torch }}, py${{ matrix.py }}, linux-x86_64, ${{ matrix.variant }}, ${{ matrix.sanitizers }})
     uses: ./.github/workflows/_build_wheel-linux.yaml
     strategy:
       fail-fast: false
       matrix:
-        torch: ['2.2.0', '2.2.1', '2.2.2']
-        py: ['3.8', '3.9', '3.10', '3.11']
-        variant: ['cpu', 'cu118', 'cu121']
+        torch: ['2.4.0', '2.4.1']
+        py: ['3.10', '3.11', '3.12']
+        variant: ['cpu', 'cu118', 'cu121', 'cu124']
         sanitizers: ['nosan']
     with:
       torch: ${{ matrix.torch }}
@@ -52,32 +49,33 @@ jobs:
       sanitizers: ${{ matrix.sanitizers }}
       version_override: ${{ inputs.version_override }}
 
-  build_pt23_wheel-linux:
+  build_pt25_wheel-linux:
     name: Build wheels (pt${{ matrix.torch }}, py${{ matrix.py }}, linux-x86_64, ${{ matrix.variant }}, ${{ matrix.sanitizers }})
     uses: ./.github/workflows/_build_wheel-linux.yaml
     strategy:
       fail-fast: false
       matrix:
-        torch: ['2.3.0']
-        py: ['3.8', '3.9', '3.10', '3.11']
-        variant: ['cpu', 'cu118', 'cu121']
+        torch: ['2.5.0', '2.5.1']
+        py: ['3.10', '3.11', '3.12']
+        variant: ['cpu', 'cu118', 'cu121', 'cu124']
         sanitizers: ['nosan']
         include:
           - run_integration_tests: false
 
           # Integration Tests
-          - torch: '2.3.0'
-            py: '3.11'
+          - torch: '2.5.1'
+            py: '3.12'
             variant: 'cpu'
             sanitizers: 'nosan'
             run_integration_tests: true
 
+          # TODO: fix lsan.out file!
           # ASAN/UBSAN
-          - torch: '2.3.0'
-            py: '3.11'
-            variant: 'cpu'
-            sanitizers: 'asan_ubsan'
-            run_integration_tests: false
+          # - torch: '2.5.1'
+          #   py: '3.12'
+          #   variant: 'cpu'
+          #   sanitizers: 'asan_ubsan'
+          #   run_integration_tests: false
     with:
       torch: ${{ matrix.torch }}
       py: ${{ matrix.py }}
@@ -86,14 +84,14 @@ jobs:
       version_override: ${{ inputs.version_override }}
       run_integration_tests: ${{ matrix.run_integration_tests }}
 
-  build_pt23_wheel-macos:
+  build_pt25_wheel-macos:
     name: Build wheels (pt${{ matrix.torch }}, py${{ matrix.py }}, macos-arm64)
     uses: ./.github/workflows/_build_wheel-macos.yaml
     strategy:
       fail-fast: false
       matrix:
-        torch: ['2.3.0']
-        py: ['3.8', '3.9', '3.10', '3.11']
+        torch: ['2.5.1']
+        py: ['3.10', '3.11', '3.12']
     with:
       torch: ${{ matrix.torch }}
       py: ${{ matrix.py }}
@@ -106,9 +104,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        torch: ['2.3.0']
-        py: ['3.8', '3.9', '3.10', '3.11']
-        variant: ['cu121']
+        torch: ['2.5.1']
+        py: ['3.10', '3.11', '3.12']
+        variant: ['cu124']
     with:
       torch: ${{ matrix.torch }}
       py: ${{ matrix.py }}
@@ -124,8 +122,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        torch: ['2.3.0']
-        py: ['3.8', '3.9', '3.10', '3.11']
+        torch: ['2.5.1']
+        py: ['3.10', '3.11', '3.12']
     with:
       torch: ${{ matrix.torch }}
       py: ${{ matrix.py }}

diff --git a/.github/workflows/_lint.yaml b/.github/workflows/_lint.yaml
@@ -7,9 +7,6 @@
 on:
   workflow_call:
 
-env:
-    ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
-
 jobs:
   lint_cc:
     name: Lint C++

diff --git a/.github/workflows/_lint_cc.yaml b/.github/workflows/_lint_cc.yaml
@@ -9,21 +9,18 @@ on:
     inputs:
       torch:
         type: string
-        default: '2.3.0'
+        default: '2.5.1'
       py:
         type: string
-        default: '3.11'
-
-env:
-    ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+        default: '3.12'
 
 jobs:
   lint:
     name: Lint
     runs-on:
       labels: 4-core-ubuntu
     container:
-      image: ghcr.io/facebookresearch/fairseq2-ci-manylinux_x86_64:2-cu118
+      image: ghcr.io/facebookresearch/fairseq2-ci-manylinux_x86_64:3-cu118
     defaults:
       run:
         shell: bash
@@ -53,10 +50,10 @@ jobs:
           export CUDAFLAGS="--allow-unsupported-compiler"
 
           # TODO: set FAIRSEQ2N_USE_CUDA once lint issues are fixed.
-          CC=clang CXX=clang++ cmake\
-             -GNinja -DFAIRSEQ2N_PYTHON_DEVEL=OFF -B build
+          CC=clang CXX=clang++ scl enable gcc-toolset-11\
+             "cmake -GNinja -DFAIRSEQ2N_PYTHON_DEVEL=OFF -B build"
 
-          cmake --build build
+          scl enable gcc-toolset-11 "cmake --build build"
       - name: Run clang-tidy
         if: success() || (failure() && steps.setup.outcome == 'success')
         working-directory: native

diff --git a/.github/workflows/_lint_py.yaml b/.github/workflows/_lint_py.yaml
@@ -9,21 +9,18 @@ on:
     inputs:
       torch:
         type: string
-        default: '2.3.0'
+        default: '2.5.1'
       py:
         type: string
-        default: '3.11'
-
-env:
-    ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+        default: '3.12'
 
 jobs:
   lint:
     name: Lint
     runs-on:
       labels: 4-core-ubuntu
     container:
-      image: ghcr.io/facebookresearch/fairseq2-ci-manylinux_x86_64:2-cpu
+      image: ghcr.io/facebookresearch/fairseq2-ci-manylinux_x86_64:3-cpu
     defaults:
       run:
         shell: bash

diff --git a/.github/workflows/_lint_sh.yaml b/.github/workflows/_lint_sh.yaml
@@ -9,18 +9,15 @@ on:
     inputs:
       py:
         type: string
-        default: '3.11'
-
-env:
-    ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+        default: '3.12'
 
 jobs:
   lint:
     name: Lint
     runs-on:
       labels: 4-core-ubuntu
     container:
-      image: ghcr.io/facebookresearch/fairseq2-ci-manylinux_x86_64:2-cpu
+      image: ghcr.io/facebookresearch/fairseq2-ci-manylinux_x86_64:3-cpu
     defaults:
       run:
         shell: bash