From c2f20e4bdab7f596485ca225de517dc096067dc8 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 11:15:13 -0500 Subject: [PATCH 01/58] ci: Attempt to split everything out Don't hate me Adam --- .github/workflows/lint.yml | 157 ++++++ .github/workflows/nf-test.yml | 270 +++++++++ .github/workflows/pytest-workflow.yml | 343 ++++++++++++ .github/workflows/test.yml | 778 -------------------------- 4 files changed, 770 insertions(+), 778 deletions(-) create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/nf-test.yml create mode 100644 .github/workflows/pytest-workflow.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000..185916f517d --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,157 @@ +name: Run Linting +on: + pull_request: + branches: [master] + merge_group: + types: [checks_requested] + branches: [master] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "self-hosted" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5 + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 + # FIXME Flip this off once we get to less than a couple hundred. Adding + # this so it will only run against changed files. It'll make it much + # easier to fix these as they come up rather than everything at once. + with: + extra_args: "" + + prettier: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Install NodeJS + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 + with: + node-version: "20" + + - name: Install Prettier + run: npm install -g prettier@3.1.0 + + - name: Run Prettier --check + run: prettier --check . + + editorconfig: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 + with: + node-version: "20" + + - name: Install editorconfig-checker + run: npm install -g editorconfig-checker + + - name: Run ECLint check + run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test) + + # FIXME Lint everything? + nf-core-lint-modules: + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + name: nf-core-lint-modules + needs: [pytest-changes, nf-test-changes] + if: ${{ (needs.pytest-changes.outputs.modules != '[]') || ( needs.nf-test-changes.outputs.modules != '[]') }} + strategy: + fail-fast: false + matrix: + tags: + [ + "${{ fromJson(needs.pytest-changes.outputs.modules) }}", + "${{ fromJson(needs.nf-test-changes.outputs.modules) }}", + ] + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Set up Python + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5 + with: + python-version: "3.11" + + - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 + id: cache-pip + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip + restore-keys: | + ${{ runner.os }}-pip + + - name: Install pip + run: python -m pip install --upgrade pip + + - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4 + with: + distribution: "temurin" + java-version: "17" + + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v2 + + - name: Install nf-core tools development version + run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev + + - name: Lint module ${{ matrix.tags }} + run: nf-core modules lint ${{ matrix.tags }} + + # FIXME Lint everything? + nf-core-lint-subworkflows: + runs-on: ubuntu-latest + name: nf-core-lint-modules + needs: [pytest-changes, nf-test-changes] + if: ${{ (needs.pytest-changes.outputs.subworkflows != '[]') || ( needs.nf-test-changes.outputs.subworkflows != '[]') }} + strategy: + fail-fast: false + matrix: + tags: + [ + "${{ fromJson(needs.pytest-changes.outputs.subworkflows) }}", + "${{ fromJson(needs.nf-test-changes.outputs.subworkflows) }}", + ] + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Set up Python + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5 + with: + python-version: "3.11" + + - name: Install pip + run: python -m pip install --upgrade pip + + - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4 + with: + distribution: "temurin" + java-version: "17" + + - name: Setup Nextflow + uses: nf-core/setup-nextflow@561fcfc7146dcb12e3871909b635ab092a781f34 # v2 + + - name: Install nf-core tools development version + run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev + + - name: Lint module ${{ matrix.tags }} + run: nf-core subworkflows lint ${{ matrix.tags }} diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml new file mode 100644 index 00000000000..764c2295b66 --- /dev/null +++ b/.github/workflows/nf-test.yml @@ -0,0 +1,270 @@ +name: Run tests +on: + pull_request: + branches: [master] + merge_group: + types: [checks_requested] + branches: [master] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "self-hosted" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + nf-test: + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + name: nf-test + needs: [nf-test-changes] + if: ( needs.nf-test-changes.outputs.paths != '[]' ) + strategy: + fail-fast: false + matrix: + path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"] + profile: [conda, docker_self_hosted, singularity] + exclude: + - path: modules/nf-core/nf-test + - profile: conda + path: modules/nf-core/angsd/gl + - profile: conda + path: modules/nf-core/annotsv/installannotations + - profile: conda + path: modules/nf-core/happy/sompy + - profile: conda + path: modules/nf-core/backsub + - profile: conda + path: modules/nf-core/bakta/bakta + - profile: conda + path: modules/nf-core/bakta/baktadbdownload + - profile: conda + path: modules/nf-core/bases2fastq + - profile: conda + path: modules/nf-core/bcl2fastq + - profile: conda + path: modules/nf-core/bclconvert + - profile: conda + path: modules/nf-core/celesta + - profile: conda + path: modules/nf-core/cellpose + - profile: conda + path: modules/nf-core/cellranger/count + - profile: conda + path: modules/nf-core/cellranger/mkfastq + - profile: conda + path: modules/nf-core/cellranger/mkgtf + - profile: conda + path: modules/nf-core/cellranger/mkref + - profile: conda + path: modules/nf-core/cellranger/mkvdjref + - profile: conda + path: modules/nf-core/cellranger/multi + - profile: conda + path: modules/nf-core/cellranger/vdj + - profile: conda + path: modules/nf-core/checkqc + - profile: conda + path: modules/nf-core/custom/dumpsoftwareversions + - profile: conda + path: modules/nf-core/deepcell/mesmer + - profile: conda + path: modules/nf-core/deepvariant + - profile: conda + path: modules/nf-core/ensemblvep/vep + - profile: conda + path: modules/nf-core/fastk/fastk + - profile: conda + path: modules/nf-core/fastk/histex + - profile: conda + path: modules/nf-core/fastk/merge + - profile: conda + path: modules/nf-core/fcs/fcsadaptor + - profile: conda + path: modules/nf-core/fcs/fcsgx + - profile: conda + path: modules/nf-core/ganon/buildcustom + - profile: conda + path: modules/nf-core/ganon/classify + - profile: conda + path: modules/nf-core/ganon/report + - profile: conda + path: modules/nf-core/ganon/table + - profile: conda + path: modules/nf-core/gatk4/cnnscorevariants + - profile: conda + path: modules/nf-core/gatk4/determinegermlinecontigploidy + - profile: conda + path: modules/nf-core/genescopefk + - profile: conda + path: modules/nf-core/ilastik/multicut + - profile: conda + path: modules/nf-core/ilastik/pixelclassification + - profile: conda + path: modules/nf-core/imputeme/vcftoprs + - profile: conda + path: modules/nf-core/mcquant + - profile: conda + path: modules/nf-core/merquryfk/katcomp + - profile: conda + path: modules/nf-core/merquryfk/katgc + - profile: conda + path: modules/nf-core/merquryfk/merquryfk + - profile: conda + path: modules/nf-core/merquryfk/ploidyplot + - profile: conda + path: modules/nf-core/molkartgarage/clahe + - profile: conda + path: modules/nf-core/quartonotebook + - profile: conda + path: modules/nf-core/scimap/spatiallda + - profile: conda + path: modules/nf-core/sentieon/bwaindex + - profile: conda + path: modules/nf-core/sentieon/bwamem + - profile: conda + path: modules/nf-core/sentieon/dedup + - profile: conda + path: modules/nf-core/sentieon/qualcal + - profile: conda + path: modules/nf-core/spaceranger/mkgtf + - profile: conda + path: modules/nf-core/spaceranger/mkref + - profile: conda + path: modules/nf-core/spaceranger/count + - profile: conda + path: modules/nf-core/spotiflow + - profile: conda + path: modules/nf-core/svanalyzer/svbenchmark + - profile: conda + path: modules/nf-core/universc + - profile: singularity + path: modules/nf-core/universc + - profile: conda + path: modules/nf-core/vt/decompose + - profile: singularity + path: modules/nf-core/bases2fastq + - profile: conda + path: modules/nf-core/wittyer + - profile: conda + path: modules/nf-core/islandpath + - profile: conda + path: subworkflows/nf-core/vcf_annotate_ensemblvep + - profile: conda + path: subworkflows/nf-core/bcl_demultiplex + - profile: conda + path: subworkflows/nf-core/fastq_align_bamcmp_bwa + - profile: conda + path: subworkflows/nf-core/fastq_align_bwa + env: + NXF_ANSI_LOG: false + NFTEST_VER: "0.9.0" + SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} + SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} + + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4 + with: + distribution: "temurin" + java-version: "17" + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v2 + + - name: Install nf-test + uses: nf-core/setup-nf-test@v1 + + - name: Setup apptainer + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-apptainer@main + + - name: Set up Singularity + if: matrix.profile == 'singularity' + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Set up Python + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5 + with: + python-version: "3.11" + + - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 + id: cache-pip-pdiff + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-pdiff + restore-keys: | + ${{ runner.os }}-pip-pdiff + + - name: Install Python dependencies + run: python -m pip install --upgrade pip pdiff cryptography + + - name: Set up miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 + with: + miniconda-version: "latest" + auto-update-conda: true + channels: conda-forge,bioconda,defaults + + - name: Conda setup + if: matrix.profile == 'conda' + run: | + conda clean -a + conda install -n base conda-libmamba-solver + conda config --set solver libmamba + echo $(realpath $CONDA)/condabin >> $GITHUB_PATH + echo $(realpath python) >> $GITHUB_PATH + + # Set up secrets + - name: Set up nextflow secrets + # TODO Only run if the tag includes `sentieon` + if: env.SENTIEON_ENCRYPTION_KEY != null && env.SENTIEON_LICENSE_MESSAGE != null + run: | + nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "${{ secrets.SENTIEON_ENCRYPTION_KEY }}" --message "${{ secrets.SENTIEON_LICENSE_MESSAGE }}") + + # Test the module + - name: Run nf-test + env: + NFT_DIFF: "pdiff" + NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" + SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} + SENTIEON_AUTH_MECH: "GitHub Actions - token" + run: | + # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker + if [ "${{ matrix.profile }}" == "docker" ]; then + PROFILE="docker_self_hosted" + else + PROFILE=${{ matrix.profile }} + fi + + NFT_WORKDIR=~ \ + nf-test test \ + --profile=${{ matrix.profile }} \ + --tap=test.tap \ + --verbose \ + ${{ matrix.path }} + + - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 + with: + path: >- + test.tap + + - name: Clean up + if: always() + run: | + sudo rm -rf /home/ubuntu/tests/ diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml new file mode 100644 index 00000000000..23522d5529b --- /dev/null +++ b/.github/workflows/pytest-workflow.yml @@ -0,0 +1,343 @@ +name: Run tests +on: + pull_request: + branches: [master] + merge_group: + types: [checks_requested] + branches: [master] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "self-hosted" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + pytest-changes: + name: pytest-changes + runs-on: ubuntu-latest + outputs: + tags: ${{ steps.filter.outputs.changes }} + modules: ${{ steps.tags.outputs.modules }} + subworkflows: ${{ steps.tags.outputs.subworkflows }} + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + with: + fetch-depth: 2 # To retrieve the preceding commit. + + # TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented + - uses: mirpedrol/paths-filter@main + id: filter + with: + filters: "tests/config/pytest_modules.yml" + token: "" + + - name: Fetch module tags + id: tags + run: | + echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT + echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT + + - name: debug + run: | + echo ${{ steps.tags.outputs.modules }} + echo ${{ steps.tags.outputs.subworkflows }} + + pytest: + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + name: pytest + needs: [pytest-changes] + if: needs.pytest-changes.outputs.tags != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"] + profile: [conda, docker, singularity] + exclude: + - tags: nf-test + - profile: conda + tags: backsub + - profile: conda + tags: bases2fastq + - profile: singularity + tags: bases2fastq + - profile: conda + tags: basicpy + - profile: conda + tags: bcl2fastq + - profile: conda + tags: bclconvert + - profile: conda + tags: bwa/aln + - profile: conda + tags: bwa/index + - profile: conda + tags: bwa/mem + - profile: conda + tags: bwa/sampe + - profile: conda + tags: bwa/samse + - profile: conda + tags: cellpose + - profile: conda + tags: cellrangerarc/count + - profile: conda + tags: cellrangerarc/mkfastq + - profile: conda + tags: cellrangerarc/mkgtf + - profile: conda + tags: cellrangerarc/mkref + - profile: conda + tags: cellrangeratac/count + - profile: conda + tags: cellrangeratac/mkfastq + - profile: conda + tags: cellrangeratac/mkref + - profile: conda + tags: checkm2/databasedownload + - profile: conda + tags: checkm2/predict + - profile: conda + tags: controlfreec/makegraph2 + - profile: conda + tags: coreograph + - profile: conda + tags: deepcell/mesmer + - profile: conda + tags: deepvariant + - profile: conda + tags: fastani + - profile: conda + tags: fastk/fastk + - profile: conda + tags: fastk/histex + - profile: conda + tags: fastk/merge + - profile: conda + tags: fcs/fcsadaptor + - profile: conda + tags: fcs/fcsgx + - profile: conda + tags: gatk4/cnnscorevariants + - profile: conda + tags: gatk4/determinegermlinecontigploidy + - profile: singularity + tags: gatk4/determinegermlinecontigploidy + - profile: conda + tags: gatk4/germlinecnvcaller + - profile: conda + tags: gatk4/postprocessgermlinecnvcalls + - profile: conda + tags: genescopefk + - profile: conda + tags: happy/sompy + - profile: conda + tags: hlala/preparegraph + - profile: conda + tags: ilastik/multicut + - profile: conda + tags: ilastik/pixelclassification + - profile: conda + tags: imputeme/vcftoprs + - profile: conda + tags: islandpath + - profile: conda + tags: manta/convertinversion + - profile: conda + tags: mcquant + - profile: conda + tags: medaka + - profile: conda + tags: merquryfk/katcomp + - profile: conda + tags: merquryfk/katgc + - profile: conda + tags: merquryfk/merquryfk + - profile: conda + tags: merquryfk/ploidyplot + - profile: conda + tags: minimap2/align + - profile: conda + tags: mitohifi/findmitoreference + - profile: conda + tags: mitohifi/mitohifi + - profile: conda + tags: nanoplot + - profile: conda + tags: ncbitools/vecscreen + - profile: conda + tags: parabricks/applybqsr + - profile: conda + tags: parabricks/dbsnp + - profile: conda + tags: parabricks/deepvariant + - profile: conda + tags: parabricks/fq2bam + - profile: conda + tags: parabricks/genotypegvcf + - profile: conda + tags: parabricks/haplotypecaller + - profile: conda + tags: parabricks/indexgvcf + - profile: conda + tags: parabricks/mutectcaller + - profile: conda + tags: picard/collecthsmetrics + - profile: conda + tags: picard/collectwgsmetrics + - profile: conda + tags: scimap/mcmicro + - profile: conda + tags: sentieon/applyvarcal + - profile: conda + tags: sentieon/datametrics + - profile: conda + tags: sentieon/dnamodelapply + - profile: conda + tags: sentieon/dnascope + - profile: conda + tags: sentieon/readwriter + - profile: conda + tags: sentieon/tnfilter + - profile: conda + tags: sentieon/tnhaplotyper2 + - profile: conda + tags: sentieon/tnscope + - profile: conda + tags: sentieon/varcal + - profile: conda + tags: sentieon/wgsmetrics + - profile: conda + tags: subworkflows/bam_qc_picard + - profile: conda + tags: subworkflows/bcl_demultiplex + - profile: conda + tags: subworkflows/fasta_clean_fcs + - profile: conda + tags: svanalyzer/svbenchmark + - profile: conda + tags: universc + - profile: singularity + tags: universc + - profile: conda + tags: vt/decompose + env: + NXF_ANSI_LOG: false + + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Set up Python + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5 + with: + python-version: "3.11" + + - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 + id: cache-pip-pytest + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-pytest + restore-keys: | + ${{ runner.os }}-pip-pytest + + - name: Install Python dependencies + run: python -m pip install --upgrade pip pytest-workflow cryptography + + - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4 + with: + distribution: "temurin" + java-version: "17" + - name: Setup Nextflow ${{ matrix.NXF_VER }} + uses: nf-core/setup-nextflow@v2 + with: + version: "${{ matrix.NXF_VER }}" + + - name: Setup apptainer + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-apptainer@main + + - name: Set up Singularity + if: matrix.profile == 'singularity' + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Set up miniconda + uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 + with: + miniconda-version: "latest" + channels: conda-forge,bioconda,defaults + python-version: ${{ matrix.python-version }} + + - name: Conda setup + run: | + conda clean -a + conda install -n base conda-libmamba-solver + conda config --set solver libmamba + echo $(realpath $CONDA)/condabin >> $GITHUB_PATH + echo $(realpath python) >> $GITHUB_PATH + + # Test the module + - name: Run pytest-workflow + # only use one thread for pytest-workflow to avoid race condition on conda cache. + run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes + + - name: Output log on failure + if: failure() + run: | + sudo apt-get update > /dev/null + sudo apt-get install bat > /dev/null + batcat --decorations=always --color=always /home/ubuntu/pytest_workflow_*/*/log.{out,err} + + - name: Setting global variables + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + id: parsed + with: + script: | + return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"') + result-encoding: string + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4 + with: + name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} + path: | + /home/ubuntu/pytest_workflow_*/*/.nextflow.log + /home/ubuntu/pytest_workflow_*/*/log.out + /home/ubuntu/pytest_workflow_*/*/log.err + /home/ubuntu/pytest_workflow_*/*/work + !/home/ubuntu/pytest_workflow_*/*/work/conda + !/home/ubuntu/pytest_workflow_*/*/work/singularity + !${{ github.workspace }}/.singularity + + confirm-pass: + runs-on: ubuntu-latest + needs: [pytest-changes, pytest] + if: always() + steps: + - name: All tests ok + if: ${{ success() || !contains(needs.*.result, 'failure') }} + run: exit 0 + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: debug-print + if: always() + run: | + echo "toJSON(needs) = ${{ toJSON(needs) }}" + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 8be6d86d95d..00000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,778 +0,0 @@ -name: Run tests -on: - pull_request: - branches: [master] - merge_group: - types: [checks_requested] - branches: [master] - workflow_dispatch: - inputs: - runners: - description: "Runners to test on" - type: choice - options: - - "ubuntu-latest" - - "self-hosted" - default: "self-hosted" - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity - NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5 - - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 - # FIXME Flip this off once we get to less than a couple hundred. Adding - # this so it will only run against changed files. It'll make it much - # easier to fix these as they come up rather than everything at once. - with: - extra_args: "--all-files" - - prettier: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: Install NodeJS - uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4 - with: - node-version: "20" - - - name: Install Prettier - run: npm install -g prettier@3.2.5 - - - name: Run Prettier --check - run: prettier --check . - - editorconfig: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4 - with: - node-version: "20" - - - name: Install editorconfig-checker - run: npm install -g editorconfig-checker - - - name: Run ECLint check - run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test) - - pytest-changes: - name: pytest-changes - runs-on: ubuntu-latest - outputs: - tags: ${{ steps.filter.outputs.changes }} - modules: ${{ steps.tags.outputs.modules }} - subworkflows: ${{ steps.tags.outputs.subworkflows }} - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - with: - fetch-depth: 2 # To retrieve the preceding commit. - - # TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented - - uses: mirpedrol/paths-filter@main - id: filter - with: - filters: "tests/config/pytest_modules.yml" - token: "" - - - name: Fetch module tags - id: tags - run: | - echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT - echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT - - - name: debug - run: | - echo ${{ steps.tags.outputs.modules }} - echo ${{ steps.tags.outputs.subworkflows }} - - nf-test-changes: - name: nf-test-changes - runs-on: ubuntu-latest - outputs: - # Expose detected tags as 'modules' and 'workflows' output variables - paths: ${{ steps.list.outputs.components }} - modules: ${{ steps.outputs.outputs.modules }} - subworkflows: ${{ steps.outputs.outputs.subworkflows}} - # Prod for version bumping - - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - with: - fetch-depth: 0 - - - name: List nf-test files - id: list - uses: adamrtalbot/detect-nf-test-changes@7c8be3ffd0d6538312b363c8c949dbbf5f26c3dd # v0.0.4 - with: - head: ${{ github.sha }} - base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} - n_parents: 2 - - - name: Separate modules and subworkflows - id: outputs - run: | - echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT - echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT - - - name: debug - run: | - echo ${{ steps.outputs.outputs.modules }} - echo ${{ steps.outputs.outputs.subworkflows }} - - nf-core-lint-modules: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} - name: nf-core-lint-modules - needs: [pytest-changes, nf-test-changes] - if: ${{ (needs.pytest-changes.outputs.modules != '[]') || ( needs.nf-test-changes.outputs.modules != '[]') }} - strategy: - fail-fast: false - matrix: - tags: - [ - "${{ fromJson(needs.pytest-changes.outputs.modules) }}", - "${{ fromJson(needs.nf-test-changes.outputs.modules) }}", - ] - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: Set up Python - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5 - with: - python-version: "3.11" - - - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 - id: cache-pip - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip - restore-keys: | - ${{ runner.os }}-pip - - - name: Install pip - run: python -m pip install --upgrade pip - - - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4 - with: - distribution: "temurin" - java-version: "17" - - - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2 - - - name: Install nf-core tools development version - run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev - - - name: Lint module ${{ matrix.tags }} - run: nf-core modules lint ${{ matrix.tags }} - - nf-core-lint-subworkflows: - runs-on: ubuntu-latest - name: nf-core-lint-modules - needs: [pytest-changes, nf-test-changes] - if: ${{ (needs.pytest-changes.outputs.subworkflows != '[]') || ( needs.nf-test-changes.outputs.subworkflows != '[]') }} - strategy: - fail-fast: false - matrix: - tags: - [ - "${{ fromJson(needs.pytest-changes.outputs.subworkflows) }}", - "${{ fromJson(needs.nf-test-changes.outputs.subworkflows) }}", - ] - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: Set up Python - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5 - with: - python-version: "3.11" - - - name: Install pip - run: python -m pip install --upgrade pip - - - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4 - with: - distribution: "temurin" - java-version: "17" - - - name: Setup Nextflow - uses: nf-core/setup-nextflow@561fcfc7146dcb12e3871909b635ab092a781f34 # v2 - - - name: Install nf-core tools development version - run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev - - - name: Lint module ${{ matrix.tags }} - run: nf-core subworkflows lint ${{ matrix.tags }} - - pytest: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} - name: pytest - needs: [pytest-changes] - if: needs.pytest-changes.outputs.tags != '[]' - strategy: - fail-fast: false - matrix: - tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"] - profile: [conda, docker, singularity] - exclude: - - tags: nf-test - - profile: conda - tags: backsub - - profile: conda - tags: bases2fastq - - profile: singularity - tags: bases2fastq - - profile: conda - tags: basicpy - - profile: conda - tags: bcl2fastq - - profile: conda - tags: bclconvert - - profile: conda - tags: bwa/aln - - profile: conda - tags: bwa/index - - profile: conda - tags: bwa/mem - - profile: conda - tags: bwa/sampe - - profile: conda - tags: bwa/samse - - profile: conda - tags: cellpose - - profile: conda - tags: cellrangerarc/count - - profile: conda - tags: cellrangerarc/mkfastq - - profile: conda - tags: cellrangerarc/mkref - - profile: conda - tags: cellrangeratac/count - - profile: conda - tags: cellrangeratac/mkfastq - - profile: conda - tags: cellrangeratac/mkref - - profile: conda - tags: checkm2/databasedownload - - profile: conda - tags: checkm2/predict - - profile: conda - tags: controlfreec/makegraph2 - - profile: conda - tags: coreograph - - profile: conda - tags: deepcell/mesmer - - profile: conda - tags: deepvariant - - profile: conda - tags: fastani - - profile: conda - tags: fastk/fastk - - profile: conda - tags: fastk/histex - - profile: conda - tags: fastk/merge - - profile: conda - tags: fcs/fcsadaptor - - profile: conda - tags: fcs/fcsgx - - profile: conda - tags: gatk4/cnnscorevariants - - profile: conda - tags: gatk4/determinegermlinecontigploidy - - profile: singularity - tags: gatk4/determinegermlinecontigploidy - - profile: conda - tags: gatk4/germlinecnvcaller - - profile: conda - tags: gatk4/postprocessgermlinecnvcalls - - profile: conda - tags: genescopefk - - profile: conda - tags: happy/sompy - - profile: conda - tags: hlala/preparegraph - - profile: conda - tags: ilastik/multicut - - profile: conda - tags: ilastik/pixelclassification - - profile: conda - tags: imputeme/vcftoprs - - profile: conda - tags: islandpath - - profile: conda - tags: manta/convertinversion - - profile: conda - tags: mcstaging/imc2mc - - profile: conda - tags: mcquant - - profile: conda - tags: medaka - - profile: conda - tags: merquryfk/katcomp - - profile: conda - tags: merquryfk/katgc - - profile: conda - tags: merquryfk/merquryfk - - profile: conda - tags: merquryfk/ploidyplot - - profile: conda - tags: minimap2/align - - profile: conda - tags: mitohifi/findmitoreference - - profile: conda - tags: mitohifi/mitohifi - - profile: conda - tags: nanoplot - - profile: conda - tags: ncbitools/vecscreen - - profile: conda - tags: parabricks/applybqsr - - profile: conda - tags: parabricks/dbsnp - - profile: conda - tags: parabricks/deepvariant - - profile: conda - tags: parabricks/fq2bam - - profile: conda - tags: parabricks/genotypegvcf - - profile: conda - tags: parabricks/haplotypecaller - - profile: conda - tags: parabricks/indexgvcf - - profile: conda - tags: parabricks/mutectcaller - - profile: conda - tags: picard/collecthsmetrics - - profile: conda - tags: picard/collectwgsmetrics - - profile: conda - tags: sentieon/applyvarcal - - profile: conda - tags: sentieon/datametrics - - profile: conda - tags: sentieon/dnamodelapply - - profile: conda - tags: sentieon/dnascope - - profile: conda - tags: sentieon/readwriter - - profile: conda - tags: sentieon/tnfilter - - profile: conda - tags: sentieon/tnhaplotyper2 - - profile: conda - tags: sentieon/tnscope - - profile: conda - tags: sentieon/varcal - - profile: conda - tags: sentieon/wgsmetrics - - profile: conda - tags: subworkflows/bam_qc_picard - - profile: conda - tags: subworkflows/bcl_demultiplex - - profile: conda - tags: subworkflows/fasta_clean_fcs - - profile: conda - tags: svanalyzer/svbenchmark - - profile: conda - tags: universc - - profile: singularity - tags: universc - - profile: conda - tags: vt/decompose - env: - NXF_ANSI_LOG: false - - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: Set up Python - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5 - with: - python-version: "3.11" - - - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 - id: cache-pip-pytest - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-pytest - restore-keys: | - ${{ runner.os }}-pip-pytest - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pytest-workflow cryptography - - - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4 - with: - distribution: "temurin" - java-version: "17" - - name: Setup Nextflow ${{ matrix.NXF_VER }} - uses: nf-core/setup-nextflow@v2 - with: - version: "${{ matrix.NXF_VER }}" - - - name: Setup apptainer - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main - - - name: Set up Singularity - if: matrix.profile == 'singularity' - run: | - mkdir -p $NXF_SINGULARITY_CACHEDIR - mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - - name: Set up miniconda - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 - with: - miniconda-version: "latest" - channels: conda-forge,bioconda - python-version: ${{ matrix.python-version }} - - - name: Conda setup - run: | - conda clean -a - conda install -n base conda-libmamba-solver - conda config --set solver libmamba - echo $(realpath $CONDA)/condabin >> $GITHUB_PATH - echo $(realpath python) >> $GITHUB_PATH - - # Test the module - - name: Run pytest-workflow - # only use one thread for pytest-workflow to avoid race condition on conda cache. - run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes - - - name: Output log on failure - if: failure() - run: | - sudo apt-get update > /dev/null - sudo apt-get install bat > /dev/null - batcat --decorations=always --color=always /home/ubuntu/pytest_workflow_*/*/log.{out,err} - - - name: Setting global variables - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 - id: parsed - with: - script: | - return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"') - result-encoding: string - - - name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4 - with: - name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} - path: | - /home/ubuntu/pytest_workflow_*/*/.nextflow.log - /home/ubuntu/pytest_workflow_*/*/log.out - /home/ubuntu/pytest_workflow_*/*/log.err - /home/ubuntu/pytest_workflow_*/*/work - !/home/ubuntu/pytest_workflow_*/*/work/conda - !/home/ubuntu/pytest_workflow_*/*/work/singularity - !${{ github.workspace }}/.singularity - - nf-test: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} - name: nf-test - needs: [nf-test-changes] - if: ( needs.nf-test-changes.outputs.paths != '[]' ) - strategy: - fail-fast: false - matrix: - path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"] - profile: [conda, docker_self_hosted, singularity] - exclude: - - path: modules/nf-core/nf-test - - profile: conda - path: modules/nf-core/angsd/gl - - profile: conda - path: modules/nf-core/annotsv/installannotations - - profile: conda - path: modules/nf-core/happy/sompy - - profile: conda - path: modules/nf-core/backsub - - profile: conda - path: modules/nf-core/bakta/bakta - - profile: conda - path: modules/nf-core/bakta/baktadbdownload - - profile: conda - path: modules/nf-core/bases2fastq - - profile: conda - path: modules/nf-core/bcl2fastq - - profile: conda - path: modules/nf-core/bclconvert - - profile: conda - path: modules/nf-core/celesta - - profile: conda - path: modules/nf-core/cellpose - - profile: conda - path: modules/nf-core/cellranger/count - - profile: conda - path: modules/nf-core/cellranger/mkfastq - - profile: conda - path: modules/nf-core/cellranger/mkgtf - - profile: conda - path: modules/nf-core/cellranger/mkref - - profile: conda - path: modules/nf-core/cellranger/mkvdjref - - profile: conda - path: modules/nf-core/cellranger/multi - - profile: conda - path: modules/nf-core/cellranger/vdj - - profile: conda - path: modules/nf-core/checkqc - - profile: conda - path: modules/nf-core/custom/dumpsoftwareversions - - profile: conda - path: modules/nf-core/deepcell/mesmer - - profile: conda - path: modules/nf-core/deepvariant - - profile: conda - path: modules/nf-core/deepvariant/callvariants - - profile: conda - path: modules/nf-core/deepvariant/makeexamples - - profile: conda - path: modules/nf-core/deepvariant/postprocessvariants - - profile: conda - path: modules/nf-core/deepvariant/rundeepvariant - - profile: conda - path: modules/nf-core/ensemblvep/vep - - profile: conda - path: modules/nf-core/fastk/fastk - - profile: conda - path: modules/nf-core/cellrangerarc/mkgtf - - profile: conda - path: modules/nf-core/fastk/histex - - profile: conda - path: modules/nf-core/fastk/merge - - profile: conda - path: modules/nf-core/fcs/fcsadaptor - - profile: conda - path: modules/nf-core/fcs/fcsgx - - profile: conda - path: modules/nf-core/ganon/buildcustom - - profile: conda - path: modules/nf-core/ganon/classify - - profile: conda - path: modules/nf-core/ganon/report - - profile: conda - path: modules/nf-core/ganon/table - - profile: conda - path: modules/nf-core/gatk4/cnnscorevariants - - profile: conda - path: modules/nf-core/gatk4/determinegermlinecontigploidy - - profile: conda - path: modules/nf-core/genescopefk - - profile: conda - path: modules/nf-core/ilastik/multicut - - profile: conda - path: modules/nf-core/ilastik/pixelclassification - - profile: conda - path: modules/nf-core/imputeme/vcftoprs - - profile: conda - path: modules/nf-core/mcstaging/imc2mc - - profile: conda - path: modules/nf-core/mcquant - - profile: conda - path: modules/nf-core/mcstaging/phenoimager2mc - - profile: conda - path: modules/nf-core/merquryfk/katcomp - - profile: conda - path: modules/nf-core/merquryfk/katgc - - profile: conda - path: modules/nf-core/merquryfk/merquryfk - - profile: conda - path: modules/nf-core/merquryfk/ploidyplot - - profile: conda - path: modules/nf-core/molkartgarage/clahe - - profile: conda - path: modules/nf-core/quartonotebook - - profile: conda - path: modules/nf-core/scimap/spatiallda - - profile: conda - path: modules/nf-core/sentieon/bwaindex - - profile: conda - path: modules/nf-core/sentieon/bwamem - - profile: conda - path: modules/nf-core/sentieon/datametrics - - profile: conda - path: modules/nf-core/sentieon/dedup - - profile: conda - path: modules/nf-core/sentieon/qualcal - - profile: conda - path: modules/nf-core/spaceranger/mkgtf - - profile: conda - path: modules/nf-core/spaceranger/mkref - - profile: conda - path: modules/nf-core/spaceranger/count - - profile: conda - path: modules/nf-core/spotiflow - - profile: conda - path: modules/nf-core/svanalyzer/svbenchmark - - profile: conda - path: modules/nf-core/universc - - profile: singularity - path: modules/nf-core/universc - - profile: conda - path: modules/nf-core/vt/decompose - - profile: singularity - path: modules/nf-core/bases2fastq - - profile: conda - path: modules/nf-core/wittyer - - profile: conda - path: modules/nf-core/islandpath - - profile: conda - path: modules/nf-core/scimap/mcmicro - - profile: conda - path: subworkflows/nf-core/vcf_annotate_ensemblvep - - profile: conda - path: subworkflows/nf-core/bcl_demultiplex - - profile: conda - path: subworkflows/nf-core/deepvariant - - profile: conda - path: subworkflows/nf-core/fastq_align_bamcmp_bwa - - profile: conda - path: subworkflows/nf-core/fastq_align_bwa - env: - NXF_ANSI_LOG: false - NFTEST_VER: "0.9.0" - SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} - SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} - - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4 - with: - distribution: "temurin" - java-version: "17" - - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2 - - - name: Install nf-test - uses: nf-core/setup-nf-test@v1 - with: - version: ${{ env.NFTEST_VER }} - - - name: Setup apptainer - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main - - - name: Set up Singularity - if: matrix.profile == 'singularity' - run: | - mkdir -p $NXF_SINGULARITY_CACHEDIR - mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - - name: Set up Python - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5 - with: - python-version: "3.11" - - - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 - id: cache-pip-pdiff - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-pdiff - restore-keys: | - ${{ runner.os }}-pip-pdiff - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pdiff cryptography - - - name: Set up miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 - with: - miniconda-version: "latest" - auto-update-conda: true - channels: conda-forge,bioconda - - - name: Conda setup - if: matrix.profile == 'conda' - run: | - conda clean -a - conda install -n base conda-libmamba-solver - conda config --set solver libmamba - echo $(realpath $CONDA)/condabin >> $GITHUB_PATH - echo $(realpath python) >> $GITHUB_PATH - - # Set up secrets - - name: Set up nextflow secrets - # TODO Only run if the tag includes `sentieon` - if: env.SENTIEON_ENCRYPTION_KEY != null && env.SENTIEON_LICENSE_MESSAGE != null - run: | - nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "${{ secrets.SENTIEON_ENCRYPTION_KEY }}" --message "${{ secrets.SENTIEON_LICENSE_MESSAGE }}") - - # Test the module - - name: Run nf-test - env: - NFT_DIFF: "pdiff" - NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" - SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} - SENTIEON_AUTH_MECH: "GitHub Actions - token" - run: | - # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker - if [ "${{ matrix.profile }}" == "docker" ]; then - PROFILE="docker_self_hosted" - else - PROFILE=${{ matrix.profile }} - fi - - NFT_WORKDIR=~ \ - nf-test test \ - --profile=${{ matrix.profile }} \ - --tap=test.tap \ - --verbose \ - ${{ matrix.path }} - - - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 - with: - path: >- - test.tap - - - name: Clean up - if: always() - run: | - sudo rm -rf /home/ubuntu/tests/ - - confirm-pass: - runs-on: ubuntu-latest - needs: - [ - prettier, - editorconfig, - pytest-changes, - nf-core-lint-modules, - nf-core-lint-subworkflows, - pytest, - nf-test-changes, - nf-test, - ] - if: always() - steps: - - name: All tests ok - if: ${{ success() || !contains(needs.*.result, 'failure') }} - run: exit 0 - - name: One or more tests failed - if: ${{ contains(needs.*.result, 'failure') }} - run: exit 1 - - - name: debug-print - if: always() - run: | - echo "toJSON(needs) = ${{ toJSON(needs) }}" - echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" From 8897f68ce5a92b4890485fae0cdcdadc4e9d8be6 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 11:22:04 -0500 Subject: [PATCH 02/58] ci: Add changed since, sharding, and ci --- .github/workflows/nf-test.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 764c2295b66..97c2383a25c 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -1,4 +1,4 @@ -name: Run tests +name: Run nf-test on: pull_request: branches: [master] @@ -29,12 +29,10 @@ jobs: nf-test: runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} name: nf-test - needs: [nf-test-changes] - if: ( needs.nf-test-changes.outputs.paths != '[]' ) strategy: fail-fast: false matrix: - path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"] + shard: [1, 2, 3] profile: [conda, docker_self_hosted, singularity] exclude: - path: modules/nf-core/nf-test @@ -171,6 +169,7 @@ jobs: env: NXF_ANSI_LOG: false NFTEST_VER: "0.9.0" + TOTAL_SHARDS: 3 SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} @@ -238,7 +237,7 @@ jobs: nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "${{ secrets.SENTIEON_ENCRYPTION_KEY }}" --message "${{ secrets.SENTIEON_LICENSE_MESSAGE }}") # Test the module - - name: Run nf-test + - name: Run Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) env: NFT_DIFF: "pdiff" NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" @@ -256,8 +255,10 @@ jobs: nf-test test \ --profile=${{ matrix.profile }} \ --tap=test.tap \ + --ci \ --verbose \ - ${{ matrix.path }} + --changed-since HEAD^ \ + --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 with: From 2e20ff4d0cf30f472305b9184b7a84c958877907 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 11:27:18 -0500 Subject: [PATCH 03/58] ci: Add filter to try to get jobs split up --- .github/workflows/nf-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 97c2383a25c..3fd89fc536b 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -34,6 +34,7 @@ jobs: matrix: shard: [1, 2, 3] profile: [conda, docker_self_hosted, singularity] + filter: [module, workflow] exclude: - path: modules/nf-core/nf-test - profile: conda @@ -258,7 +259,8 @@ jobs: --ci \ --verbose \ --changed-since HEAD^ \ - --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} + --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ + --filter ${{ matrix.filter }} - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 with: From ea0de231fccfc9c7beaaea27395df6e7d0873e9e Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 11:27:54 -0500 Subject: [PATCH 04/58] ci: Switch to only-changed --- .github/workflows/nf-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 3fd89fc536b..d733aa9622d 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -258,7 +258,7 @@ jobs: --tap=test.tap \ --ci \ --verbose \ - --changed-since HEAD^ \ + --only-changed \ --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ --filter ${{ matrix.filter }} From 2c5b4bd0a0c361ad44599414cace8ec41ea65513 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 11:29:19 -0500 Subject: [PATCH 05/58] ci: See if follow-dependencies works without "related-tests" --- .github/workflows/nf-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index d733aa9622d..091d33af97f 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -260,7 +260,8 @@ jobs: --verbose \ --only-changed \ --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ - --filter ${{ matrix.filter }} + --filter ${{ matrix.filter }} \ + --follow-dependencies - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 with: From 6322086e54dd63ed8328cc0d7234010babd212f0 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 11:40:01 -0500 Subject: [PATCH 06/58] ci: Remove skipped tests Idk what we're going to do about these... --- .github/workflows/nf-test.yml | 265 +++++++++++++++++----------------- 1 file changed, 133 insertions(+), 132 deletions(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 091d33af97f..c1856e0189b 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -35,138 +35,6 @@ jobs: shard: [1, 2, 3] profile: [conda, docker_self_hosted, singularity] filter: [module, workflow] - exclude: - - path: modules/nf-core/nf-test - - profile: conda - path: modules/nf-core/angsd/gl - - profile: conda - path: modules/nf-core/annotsv/installannotations - - profile: conda - path: modules/nf-core/happy/sompy - - profile: conda - path: modules/nf-core/backsub - - profile: conda - path: modules/nf-core/bakta/bakta - - profile: conda - path: modules/nf-core/bakta/baktadbdownload - - profile: conda - path: modules/nf-core/bases2fastq - - profile: conda - path: modules/nf-core/bcl2fastq - - profile: conda - path: modules/nf-core/bclconvert - - profile: conda - path: modules/nf-core/celesta - - profile: conda - path: modules/nf-core/cellpose - - profile: conda - path: modules/nf-core/cellranger/count - - profile: conda - path: modules/nf-core/cellranger/mkfastq - - profile: conda - path: modules/nf-core/cellranger/mkgtf - - profile: conda - path: modules/nf-core/cellranger/mkref - - profile: conda - path: modules/nf-core/cellranger/mkvdjref - - profile: conda - path: modules/nf-core/cellranger/multi - - profile: conda - path: modules/nf-core/cellranger/vdj - - profile: conda - path: modules/nf-core/checkqc - - profile: conda - path: modules/nf-core/custom/dumpsoftwareversions - - profile: conda - path: modules/nf-core/deepcell/mesmer - - profile: conda - path: modules/nf-core/deepvariant - - profile: conda - path: modules/nf-core/ensemblvep/vep - - profile: conda - path: modules/nf-core/fastk/fastk - - profile: conda - path: modules/nf-core/fastk/histex - - profile: conda - path: modules/nf-core/fastk/merge - - profile: conda - path: modules/nf-core/fcs/fcsadaptor - - profile: conda - path: modules/nf-core/fcs/fcsgx - - profile: conda - path: modules/nf-core/ganon/buildcustom - - profile: conda - path: modules/nf-core/ganon/classify - - profile: conda - path: modules/nf-core/ganon/report - - profile: conda - path: modules/nf-core/ganon/table - - profile: conda - path: modules/nf-core/gatk4/cnnscorevariants - - profile: conda - path: modules/nf-core/gatk4/determinegermlinecontigploidy - - profile: conda - path: modules/nf-core/genescopefk - - profile: conda - path: modules/nf-core/ilastik/multicut - - profile: conda - path: modules/nf-core/ilastik/pixelclassification - - profile: conda - path: modules/nf-core/imputeme/vcftoprs - - profile: conda - path: modules/nf-core/mcquant - - profile: conda - path: modules/nf-core/merquryfk/katcomp - - profile: conda - path: modules/nf-core/merquryfk/katgc - - profile: conda - path: modules/nf-core/merquryfk/merquryfk - - profile: conda - path: modules/nf-core/merquryfk/ploidyplot - - profile: conda - path: modules/nf-core/molkartgarage/clahe - - profile: conda - path: modules/nf-core/quartonotebook - - profile: conda - path: modules/nf-core/scimap/spatiallda - - profile: conda - path: modules/nf-core/sentieon/bwaindex - - profile: conda - path: modules/nf-core/sentieon/bwamem - - profile: conda - path: modules/nf-core/sentieon/dedup - - profile: conda - path: modules/nf-core/sentieon/qualcal - - profile: conda - path: modules/nf-core/spaceranger/mkgtf - - profile: conda - path: modules/nf-core/spaceranger/mkref - - profile: conda - path: modules/nf-core/spaceranger/count - - profile: conda - path: modules/nf-core/spotiflow - - profile: conda - path: modules/nf-core/svanalyzer/svbenchmark - - profile: conda - path: modules/nf-core/universc - - profile: singularity - path: modules/nf-core/universc - - profile: conda - path: modules/nf-core/vt/decompose - - profile: singularity - path: modules/nf-core/bases2fastq - - profile: conda - path: modules/nf-core/wittyer - - profile: conda - path: modules/nf-core/islandpath - - profile: conda - path: subworkflows/nf-core/vcf_annotate_ensemblvep - - profile: conda - path: subworkflows/nf-core/bcl_demultiplex - - profile: conda - path: subworkflows/nf-core/fastq_align_bamcmp_bwa - - profile: conda - path: subworkflows/nf-core/fastq_align_bwa env: NXF_ANSI_LOG: false NFTEST_VER: "0.9.0" @@ -272,3 +140,136 @@ jobs: if: always() run: | sudo rm -rf /home/ubuntu/tests/ + +# TODO What do we do with these? +# - path: modules/nf-core/nf-test +# - profile: conda +# path: modules/nf-core/angsd/gl +# - profile: conda +# path: modules/nf-core/annotsv/installannotations +# - profile: conda +# path: modules/nf-core/happy/sompy +# - profile: conda +# path: modules/nf-core/backsub +# - profile: conda +# path: modules/nf-core/bakta/bakta +# - profile: conda +# path: modules/nf-core/bakta/baktadbdownload +# - profile: conda +# path: modules/nf-core/bases2fastq +# - profile: conda +# path: modules/nf-core/bcl2fastq +# - profile: conda +# path: modules/nf-core/bclconvert +# - profile: conda +# path: modules/nf-core/celesta +# - profile: conda +# path: modules/nf-core/cellpose +# - profile: conda +# path: modules/nf-core/cellranger/count +# - profile: conda +# path: modules/nf-core/cellranger/mkfastq +# - profile: conda +# path: modules/nf-core/cellranger/mkgtf +# - profile: conda +# path: modules/nf-core/cellranger/mkref +# - profile: conda +# path: modules/nf-core/cellranger/mkvdjref +# - profile: conda +# path: modules/nf-core/cellranger/multi +# - profile: conda +# path: modules/nf-core/cellranger/vdj +# - profile: conda +# path: modules/nf-core/checkqc +# - profile: conda +# path: modules/nf-core/custom/dumpsoftwareversions +# - profile: conda +# path: modules/nf-core/deepcell/mesmer +# - profile: conda +# path: modules/nf-core/deepvariant +# - profile: conda +# path: modules/nf-core/ensemblvep/vep +# - profile: conda +# path: modules/nf-core/fastk/fastk +# - profile: conda +# path: modules/nf-core/fastk/histex +# - profile: conda +# path: modules/nf-core/fastk/merge +# - profile: conda +# path: modules/nf-core/fcs/fcsadaptor +# - profile: conda +# path: modules/nf-core/fcs/fcsgx +# - profile: conda +# path: modules/nf-core/ganon/buildcustom +# - profile: conda +# path: modules/nf-core/ganon/classify +# - profile: conda +# path: modules/nf-core/ganon/report +# - profile: conda +# path: modules/nf-core/ganon/table +# - profile: conda +# path: modules/nf-core/gatk4/cnnscorevariants +# - profile: conda +# path: modules/nf-core/gatk4/determinegermlinecontigploidy +# - profile: conda +# path: modules/nf-core/genescopefk +# - profile: conda +# path: modules/nf-core/ilastik/multicut +# - profile: conda +# path: modules/nf-core/ilastik/pixelclassification +# - profile: conda +# path: modules/nf-core/imputeme/vcftoprs +# - profile: conda +# path: modules/nf-core/mcquant +# - profile: conda +# path: modules/nf-core/merquryfk/katcomp +# - profile: conda +# path: modules/nf-core/merquryfk/katgc +# - profile: conda +# path: modules/nf-core/merquryfk/merquryfk +# - profile: conda +# path: modules/nf-core/merquryfk/ploidyplot +# - profile: conda +# path: modules/nf-core/molkartgarage/clahe +# - profile: conda +# path: modules/nf-core/quartonotebook +# - profile: conda +# path: modules/nf-core/scimap/spatiallda +# - profile: conda +# path: modules/nf-core/sentieon/bwaindex +# - profile: conda +# path: modules/nf-core/sentieon/bwamem +# - profile: conda +# path: modules/nf-core/sentieon/dedup +# - profile: conda +# path: modules/nf-core/sentieon/qualcal +# - profile: conda +# path: modules/nf-core/spaceranger/mkgtf +# - profile: conda +# path: modules/nf-core/spaceranger/mkref +# - profile: conda +# path: modules/nf-core/spaceranger/count +# - profile: conda +# path: modules/nf-core/spotiflow +# - profile: conda +# path: modules/nf-core/svanalyzer/svbenchmark +# - profile: conda +# path: modules/nf-core/universc +# - profile: singularity +# path: modules/nf-core/universc +# - profile: conda +# path: modules/nf-core/vt/decompose +# - profile: singularity +# path: modules/nf-core/bases2fastq +# - profile: conda +# path: modules/nf-core/wittyer +# - profile: conda +# path: modules/nf-core/islandpath +# - profile: conda +# path: subworkflows/nf-core/vcf_annotate_ensemblvep +# - profile: conda +# path: subworkflows/nf-core/bcl_demultiplex +# - profile: conda +# path: subworkflows/nf-core/fastq_align_bamcmp_bwa +# - profile: conda +# path: subworkflows/nf-core/fastq_align_bwa From 71d9e5aa851447e375d6a33dd8663aece932cb47 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 11:49:32 -0500 Subject: [PATCH 07/58] ci: Actually use the nf-test version --- .github/workflows/nf-test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index c1856e0189b..57f3604c316 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -37,6 +37,7 @@ jobs: filter: [module, workflow] env: NXF_ANSI_LOG: false + NXF_VER: "24.04.4" NFTEST_VER: "0.9.0" TOTAL_SHARDS: 3 SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} @@ -51,9 +52,13 @@ jobs: java-version: "17" - name: Setup Nextflow uses: nf-core/setup-nextflow@v2 + with: + version: ${{ env.NXF_VER }} - name: Install nf-test uses: nf-core/setup-nf-test@v1 + with: + version: ${{ env.NFTEST_VER }} - name: Setup apptainer if: matrix.profile == 'singularity' From afd049de7ff8cf8be963065a8cd8ffec9580eb0f Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 11:56:14 -0500 Subject: [PATCH 08/58] ci: module => process --- .github/workflows/nf-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 57f3604c316..1150a8c53e0 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -34,7 +34,7 @@ jobs: matrix: shard: [1, 2, 3] profile: [conda, docker_self_hosted, singularity] - filter: [module, workflow] + filter: [process, workflow] env: NXF_ANSI_LOG: false NXF_VER: "24.04.4" From a823ee379cc2a49da63c2bd748ef8f3fef8bd53d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 12:04:37 -0500 Subject: [PATCH 09/58] ci: Clean up job names --- .github/workflows/nf-test.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 1150a8c53e0..c0ebe3af253 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -27,8 +27,8 @@ env: jobs: nf-test: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} - name: nf-test + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + name: ${{ matrix.profile }}\|${{ matrix.filter }}\|Shard ${{ matrix.shard }} strategy: fail-fast: false matrix: @@ -111,7 +111,7 @@ jobs: nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "${{ secrets.SENTIEON_ENCRYPTION_KEY }}" --message "${{ secrets.SENTIEON_LICENSE_MESSAGE }}") # Test the module - - name: Run Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) + - name: Run Tests (Shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }}) env: NFT_DIFF: "pdiff" NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" @@ -136,6 +136,8 @@ jobs: --filter ${{ matrix.filter }} \ --follow-dependencies + # TODO If no test.tap, then make one to spoof? + - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 with: path: >- @@ -147,6 +149,7 @@ jobs: sudo rm -rf /home/ubuntu/tests/ # TODO What do we do with these? +# I think we can require Docker,modules,Shard 1..Singularity,subworkflows,3 and skip the condas now # - path: modules/nf-core/nf-test # - profile: conda # path: modules/nf-core/angsd/gl From 0d2c26c9da948dce3a5cffcb9c1eddf04beb3266 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 12:06:31 -0500 Subject: [PATCH 10/58] dummy: Make a change --- modules/nf-core/bowtie/align/main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index 5e72b02a678..42939e51586 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -29,6 +29,7 @@ process BOWTIE_ALIGN { def endedness = meta.single_end ? "$reads" : "-1 ${reads[0]} -2 ${reads[1]}" """ INDEX=\$(find -L ./ -name "*.3.ebwt" | sed 's/\\.3.ebwt\$//') + bowtie \\ --threads $task.cpus \\ --sam \\ From 97f64066ff1834bd441c6dd71881613948543677 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 12:11:20 -0500 Subject: [PATCH 11/58] ci: Skip test.tap --- .github/workflows/nf-test.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index c0ebe3af253..5b9ba302f01 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -136,12 +136,11 @@ jobs: --filter ${{ matrix.filter }} \ --follow-dependencies - # TODO If no test.tap, then make one to spoof? - - - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 - with: - path: >- - test.tap + # TODO If no test.tap, then make one to spoof? + # - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 + # with: + # path: >- + # test.tap - name: Clean up if: always() From c486922ad5573c8e7e03412c1cc9a868b85d48da Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 12:12:30 -0500 Subject: [PATCH 12/58] ci: Add fetch-depth --- .github/workflows/nf-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 5b9ba302f01..38aeae2974c 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -45,6 +45,8 @@ jobs: steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + with: + fetch-depth: 0 - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4 with: From d58ecdbfd35a3612f24a7af436aac3563f546774 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 12:29:30 -0500 Subject: [PATCH 13/58] ci: Clean up name --- .github/workflows/nf-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 38aeae2974c..065a16108dd 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -28,7 +28,7 @@ env: jobs: nf-test: runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} - name: ${{ matrix.profile }}\|${{ matrix.filter }}\|Shard ${{ matrix.shard }} + name: ${{ matrix.profile }}|${{ matrix.filter }}|${{ matrix.shard }} strategy: fail-fast: false matrix: From 52a826d329ad890ba8808f08d58dab3ed92f0281 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 12:36:20 -0500 Subject: [PATCH 14/58] ci: Lint everything --- .github/workflows/lint.yml | 32 +++++---------------------- .github/workflows/pytest-workflow.yml | 2 +- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 185916f517d..38e7f4e1721 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -70,20 +70,11 @@ jobs: - name: Run ECLint check run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test) - # FIXME Lint everything? nf-core-lint-modules: runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} - name: nf-core-lint-modules - needs: [pytest-changes, nf-test-changes] - if: ${{ (needs.pytest-changes.outputs.modules != '[]') || ( needs.nf-test-changes.outputs.modules != '[]') }} + name: nf-core lint modules strategy: fail-fast: false - matrix: - tags: - [ - "${{ fromJson(needs.pytest-changes.outputs.modules) }}", - "${{ fromJson(needs.nf-test-changes.outputs.modules) }}", - ] steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 @@ -114,23 +105,12 @@ jobs: - name: Install nf-core tools development version run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev - - name: Lint module ${{ matrix.tags }} - run: nf-core modules lint ${{ matrix.tags }} + - name: Lint modules + run: nf-core modules lint -a - # FIXME Lint everything? nf-core-lint-subworkflows: runs-on: ubuntu-latest - name: nf-core-lint-modules - needs: [pytest-changes, nf-test-changes] - if: ${{ (needs.pytest-changes.outputs.subworkflows != '[]') || ( needs.nf-test-changes.outputs.subworkflows != '[]') }} - strategy: - fail-fast: false - matrix: - tags: - [ - "${{ fromJson(needs.pytest-changes.outputs.subworkflows) }}", - "${{ fromJson(needs.nf-test-changes.outputs.subworkflows) }}", - ] + name: nf-core lint subworkflows steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 @@ -153,5 +133,5 @@ jobs: - name: Install nf-core tools development version run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev - - name: Lint module ${{ matrix.tags }} - run: nf-core subworkflows lint ${{ matrix.tags }} + - name: Lint subworkflows + run: nf-core subworkflows lint -a diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 23522d5529b..de04a1570a8 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -1,4 +1,4 @@ -name: Run tests +name: Run pytest-workflow on: pull_request: branches: [master] From 1043d64b38f6a5253cc40754f5ca1ea83d7ce4a7 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 12:42:15 -0500 Subject: [PATCH 15/58] ci: Get the job names clean --- .github/workflows/nf-test.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 065a16108dd..4a87534aaed 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -28,13 +28,20 @@ env: jobs: nf-test: runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} - name: ${{ matrix.profile }}|${{ matrix.filter }}|${{ matrix.shard }} + # NOTE I think this is the cleanest way to get them organized + # process | conda | 1 + # process | conda | 2 + # process | conda | 3 + # process | docker_self_hosted | 1 + # ... + # workflow | singularity | 3 + name: ${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.shard }} strategy: fail-fast: false matrix: - shard: [1, 2, 3] - profile: [conda, docker_self_hosted, singularity] filter: [process, workflow] + profile: [conda, docker_self_hosted, singularity] + shard: [1, 2, 3] env: NXF_ANSI_LOG: false NXF_VER: "24.04.4" From 94c245387efd0d7d4cea6e91b09ed2f13186f94c Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 23 Aug 2024 14:16:16 -0500 Subject: [PATCH 16/58] ci: Add hide-progress on linting --- .github/workflows/lint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 38e7f4e1721..d9e8215f3e7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -106,7 +106,7 @@ jobs: run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev - name: Lint modules - run: nf-core modules lint -a + run: nf-core --hide-progress modules lint -a nf-core-lint-subworkflows: runs-on: ubuntu-latest @@ -134,4 +134,4 @@ jobs: run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev - name: Lint subworkflows - run: nf-core subworkflows lint -a + run: nf-core --hide-progress subworkflows lint -a From ab5194b3476f7a5dafb90cd4836b03710fa6e306 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 24 Aug 2024 17:19:01 -0500 Subject: [PATCH 17/58] ci: Add psuedocode for conda-fail.yml https://nfcore.slack.com/archives/CJRH30T6V/p1724406283145319 --- .github/workflows/nf-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 4a87534aaed..69d95b7d2e0 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -51,6 +51,8 @@ jobs: SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} steps: + # TODO If conda-fail.yml exists and matrix = conda skip + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 with: fetch-depth: 0 @@ -145,6 +147,8 @@ jobs: --filter ${{ matrix.filter }} \ --follow-dependencies + # TODO If matrix == conda create a conda-fail.yml and commit it + # TODO If no test.tap, then make one to spoof? # - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 # with: From 2a218e3da33ae9647fdb0fe078fc0e4f85e3c9ef Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 10 Sep 2024 11:49:33 -0500 Subject: [PATCH 18/58] test: Snapshot the versions contents, not the hash https://github.com/nf-core/modules/issues/6505 --- modules/nf-core/bowtie/build/tests/main.nf.test | 4 +++- .../nf-core/bowtie/build/tests/main.nf.test.snap | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/modules/nf-core/bowtie/build/tests/main.nf.test b/modules/nf-core/bowtie/build/tests/main.nf.test index 25fb3dad83b..b58f1d69a2f 100644 --- a/modules/nf-core/bowtie/build/tests/main.nf.test +++ b/modules/nf-core/bowtie/build/tests/main.nf.test @@ -34,6 +34,7 @@ nextflow_process { test("sarscov2 - fasta - stub") { options "-stub" + tag "version" when { process { @@ -48,7 +49,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(process.out).match() }, + { assert snapshot(path(process.out.versions.get(0)).yaml).match("versions") }, ) } diff --git a/modules/nf-core/bowtie/build/tests/main.nf.test.snap b/modules/nf-core/bowtie/build/tests/main.nf.test.snap index e8061756ba2..6951ac5b5d4 100644 --- a/modules/nf-core/bowtie/build/tests/main.nf.test.snap +++ b/modules/nf-core/bowtie/build/tests/main.nf.test.snap @@ -46,6 +46,20 @@ }, "timestamp": "2024-06-18T08:38:14.852528155" }, + "versions": { + "content": [ + { + "BOWTIE_BUILD": { + "bowtie": "1.3.0" + } + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-09-10T11:48:00.007395903" + }, "sarscov2 - fasta": { "content": [ { From e833cd5b6057a66c3258f8c9d787ba411916534d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 18 Sep 2024 13:35:34 -0500 Subject: [PATCH 19/58] ci: Keep running nf-core lint the way it was Blocked by https://github.com/nf-core/tools/issues/3140 --- .github/workflows/lint.yml | 55 +++++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d9e8215f3e7..bb20af0f209 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -70,16 +70,53 @@ jobs: - name: Run ECLint check run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test) + ################### + # nf-core linting # + ################### + nf-core-changes: + name: nf-core-changes + runs-on: ubuntu-latest + outputs: + tags: ${{ steps.filter.outputs.changes }} + modules: ${{ steps.tags.outputs.modules }} + subworkflows: ${{ steps.tags.outputs.subworkflows }} + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + with: + fetch-depth: 2 # To retrieve the preceding commit. + + # TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented + - uses: mirpedrol/paths-filter@main + id: filter + with: + filters: "tests/config/pytest_modules.yml" + token: "" + + - name: Fetch module tags + id: tags + run: | + echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT + echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT + + - name: debug + run: | + echo ${{ steps.tags.outputs.modules }} + echo ${{ steps.tags.outputs.subworkflows }} + nf-core-lint-modules: runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} name: nf-core lint modules + needs: nf-core-changes + if: ${{ (needs.nf-core-changes.outputs.modules != '[]') }} strategy: fail-fast: false + matrix: + tags: "${{ fromJson(needs.nf-core-changes.outputs.modules) }}" steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Set up Python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5 with: python-version: "3.11" @@ -105,17 +142,23 @@ jobs: - name: Install nf-core tools development version run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev - - name: Lint modules - run: nf-core --hide-progress modules lint -a + - name: Lint module ${{ matrix.tags }} + run: nf-core modules lint ${{ matrix.tags }} nf-core-lint-subworkflows: runs-on: ubuntu-latest name: nf-core lint subworkflows + needs: nf-core-changes + if: ${{ (needs.nf-core-changes.outputs.subworkflows != '[]') }} + strategy: + fail-fast: false + matrix: + tags: "${{ fromJson(needs.nf-core-changes.outputs.subworkflows) }}" steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - name: Set up Python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5 with: python-version: "3.11" @@ -133,5 +176,5 @@ jobs: - name: Install nf-core tools development version run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev - - name: Lint subworkflows - run: nf-core --hide-progress subworkflows lint -a + - name: Lint module ${{ matrix.tags }} + run: nf-core subworkflows lint ${{ matrix.tags }} From 392ffe78e4a2ea42ade220cf0d611799c5275797 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 24 Sep 2024 07:26:18 -0500 Subject: [PATCH 20/58] ci: Move conda skips out --- .github/conda_skip.yml | 133 +++++++++++++++++++++++++++++++++ .github/workflows/nf-test.yml | 134 ---------------------------------- 2 files changed, 133 insertions(+), 134 deletions(-) create mode 100644 .github/conda_skip.yml diff --git a/.github/conda_skip.yml b/.github/conda_skip.yml new file mode 100644 index 00000000000..44e43a740a4 --- /dev/null +++ b/.github/conda_skip.yml @@ -0,0 +1,133 @@ +# TODO What do we do with these? +# I think we can require Docker,modules,Shard 1..Singularity,subworkflows,3 and skip the condas now +# - path: modules/nf-core/nf-test +# - profile: conda +# path: modules/nf-core/angsd/gl +# - profile: conda +# path: modules/nf-core/annotsv/installannotations +# - profile: conda +# path: modules/nf-core/happy/sompy +# - profile: conda +# path: modules/nf-core/backsub +# - profile: conda +# path: modules/nf-core/bakta/bakta +# - profile: conda +# path: modules/nf-core/bakta/baktadbdownload +# - profile: conda +# path: modules/nf-core/bases2fastq +# - profile: conda +# path: modules/nf-core/bcl2fastq +# - profile: conda +# path: modules/nf-core/bclconvert +# - profile: conda +# path: modules/nf-core/celesta +# - profile: conda +# path: modules/nf-core/cellpose +# - profile: conda +# path: modules/nf-core/cellranger/count +# - profile: conda +# path: modules/nf-core/cellranger/mkfastq +# - profile: conda +# path: modules/nf-core/cellranger/mkgtf +# - profile: conda +# path: modules/nf-core/cellranger/mkref +# - profile: conda +# path: modules/nf-core/cellranger/mkvdjref +# - profile: conda +# path: modules/nf-core/cellranger/multi +# - profile: conda +# path: modules/nf-core/cellranger/vdj +# - profile: conda +# path: modules/nf-core/checkqc +# - profile: conda +# path: modules/nf-core/custom/dumpsoftwareversions +# - profile: conda +# path: modules/nf-core/deepcell/mesmer +# - profile: conda +# path: modules/nf-core/deepvariant +# - profile: conda +# path: modules/nf-core/ensemblvep/vep +# - profile: conda +# path: modules/nf-core/fastk/fastk +# - profile: conda +# path: modules/nf-core/fastk/histex +# - profile: conda +# path: modules/nf-core/fastk/merge +# - profile: conda +# path: modules/nf-core/fcs/fcsadaptor +# - profile: conda +# path: modules/nf-core/fcs/fcsgx +# - profile: conda +# path: modules/nf-core/ganon/buildcustom +# - profile: conda +# path: modules/nf-core/ganon/classify +# - profile: conda +# path: modules/nf-core/ganon/report +# - profile: conda +# path: modules/nf-core/ganon/table +# - profile: conda +# path: modules/nf-core/gatk4/cnnscorevariants +# - profile: conda +# path: modules/nf-core/gatk4/determinegermlinecontigploidy +# - profile: conda +# path: modules/nf-core/genescopefk +# - profile: conda +# path: modules/nf-core/ilastik/multicut +# - profile: conda +# path: modules/nf-core/ilastik/pixelclassification +# - profile: conda +# path: modules/nf-core/imputeme/vcftoprs +# - profile: conda +# path: modules/nf-core/mcquant +# - profile: conda +# path: modules/nf-core/merquryfk/katcomp +# - profile: conda +# path: modules/nf-core/merquryfk/katgc +# - profile: conda +# path: modules/nf-core/merquryfk/merquryfk +# - profile: conda +# path: modules/nf-core/merquryfk/ploidyplot +# - profile: conda +# path: modules/nf-core/molkartgarage/clahe +# - profile: conda +# path: modules/nf-core/quartonotebook +# - profile: conda +# path: modules/nf-core/scimap/spatiallda +# - profile: conda +# path: modules/nf-core/sentieon/bwaindex +# - profile: conda +# path: modules/nf-core/sentieon/bwamem +# - profile: conda +# path: modules/nf-core/sentieon/dedup +# - profile: conda +# path: modules/nf-core/sentieon/qualcal +# - profile: conda +# path: modules/nf-core/spaceranger/mkgtf +# - profile: conda +# path: modules/nf-core/spaceranger/mkref +# - profile: conda +# path: modules/nf-core/spaceranger/count +# - profile: conda +# path: modules/nf-core/spotiflow +# - profile: conda +# path: modules/nf-core/svanalyzer/svbenchmark +# - profile: conda +# path: modules/nf-core/universc +# - profile: singularity +# path: modules/nf-core/universc +# - profile: conda +# path: modules/nf-core/vt/decompose +# - profile: singularity +# path: modules/nf-core/bases2fastq +# - profile: conda +# path: modules/nf-core/wittyer +# - profile: conda +# path: modules/nf-core/islandpath +# - profile: conda +# path: subworkflows/nf-core/vcf_annotate_ensemblvep +# - profile: conda +# path: subworkflows/nf-core/bcl_demultiplex +# - profile: conda +# path: subworkflows/nf-core/fastq_align_bamcmp_bwa +# - profile: conda +# path: subworkflows/nf-core/fastq_align_bwa diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 69d95b7d2e0..f38e2ceb35c 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -159,137 +159,3 @@ jobs: if: always() run: | sudo rm -rf /home/ubuntu/tests/ - -# TODO What do we do with these? -# I think we can require Docker,modules,Shard 1..Singularity,subworkflows,3 and skip the condas now -# - path: modules/nf-core/nf-test -# - profile: conda -# path: modules/nf-core/angsd/gl -# - profile: conda -# path: modules/nf-core/annotsv/installannotations -# - profile: conda -# path: modules/nf-core/happy/sompy -# - profile: conda -# path: modules/nf-core/backsub -# - profile: conda -# path: modules/nf-core/bakta/bakta -# - profile: conda -# path: modules/nf-core/bakta/baktadbdownload -# - profile: conda -# path: modules/nf-core/bases2fastq -# - profile: conda -# path: modules/nf-core/bcl2fastq -# - profile: conda -# path: modules/nf-core/bclconvert -# - profile: conda -# path: modules/nf-core/celesta -# - profile: conda -# path: modules/nf-core/cellpose -# - profile: conda -# path: modules/nf-core/cellranger/count -# - profile: conda -# path: modules/nf-core/cellranger/mkfastq -# - profile: conda -# path: modules/nf-core/cellranger/mkgtf -# - profile: conda -# path: modules/nf-core/cellranger/mkref -# - profile: conda -# path: modules/nf-core/cellranger/mkvdjref -# - profile: conda -# path: modules/nf-core/cellranger/multi -# - profile: conda -# path: modules/nf-core/cellranger/vdj -# - profile: conda -# path: modules/nf-core/checkqc -# - profile: conda -# path: modules/nf-core/custom/dumpsoftwareversions -# - profile: conda -# path: modules/nf-core/deepcell/mesmer -# - profile: conda -# path: modules/nf-core/deepvariant -# - profile: conda -# path: modules/nf-core/ensemblvep/vep -# - profile: conda -# path: modules/nf-core/fastk/fastk -# - profile: conda -# path: modules/nf-core/fastk/histex -# - profile: conda -# path: modules/nf-core/fastk/merge -# - profile: conda -# path: modules/nf-core/fcs/fcsadaptor -# - profile: conda -# path: modules/nf-core/fcs/fcsgx -# - profile: conda -# path: modules/nf-core/ganon/buildcustom -# - profile: conda -# path: modules/nf-core/ganon/classify -# - profile: conda -# path: modules/nf-core/ganon/report -# - profile: conda -# path: modules/nf-core/ganon/table -# - profile: conda -# path: modules/nf-core/gatk4/cnnscorevariants -# - profile: conda -# path: modules/nf-core/gatk4/determinegermlinecontigploidy -# - profile: conda -# path: modules/nf-core/genescopefk -# - profile: conda -# path: modules/nf-core/ilastik/multicut -# - profile: conda -# path: modules/nf-core/ilastik/pixelclassification -# - profile: conda -# path: modules/nf-core/imputeme/vcftoprs -# - profile: conda -# path: modules/nf-core/mcquant -# - profile: conda -# path: modules/nf-core/merquryfk/katcomp -# - profile: conda -# path: modules/nf-core/merquryfk/katgc -# - profile: conda -# path: modules/nf-core/merquryfk/merquryfk -# - profile: conda -# path: modules/nf-core/merquryfk/ploidyplot -# - profile: conda -# path: modules/nf-core/molkartgarage/clahe -# - profile: conda -# path: modules/nf-core/quartonotebook -# - profile: conda -# path: modules/nf-core/scimap/spatiallda -# - profile: conda -# path: modules/nf-core/sentieon/bwaindex -# - profile: conda -# path: modules/nf-core/sentieon/bwamem -# - profile: conda -# path: modules/nf-core/sentieon/dedup -# - profile: conda -# path: modules/nf-core/sentieon/qualcal -# - profile: conda -# path: modules/nf-core/spaceranger/mkgtf -# - profile: conda -# path: modules/nf-core/spaceranger/mkref -# - profile: conda -# path: modules/nf-core/spaceranger/count -# - profile: conda -# path: modules/nf-core/spotiflow -# - profile: conda -# path: modules/nf-core/svanalyzer/svbenchmark -# - profile: conda -# path: modules/nf-core/universc -# - profile: singularity -# path: modules/nf-core/universc -# - profile: conda -# path: modules/nf-core/vt/decompose -# - profile: singularity -# path: modules/nf-core/bases2fastq -# - profile: conda -# path: modules/nf-core/wittyer -# - profile: conda -# path: modules/nf-core/islandpath -# - profile: conda -# path: subworkflows/nf-core/vcf_annotate_ensemblvep -# - profile: conda -# path: subworkflows/nf-core/bcl_demultiplex -# - profile: conda -# path: subworkflows/nf-core/fastq_align_bamcmp_bwa -# - profile: conda -# path: subworkflows/nf-core/fastq_align_bwa From af2d9177793bf48c872addadf6e22dd99e0123bc Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 24 Sep 2024 07:36:57 -0500 Subject: [PATCH 21/58] ci: Address a comment --- .github/workflows/nf-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index f38e2ceb35c..a6acf4f0283 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -52,6 +52,7 @@ jobs: steps: # TODO If conda-fail.yml exists and matrix = conda skip + # https://github.com/askimed/nf-test/issues/260 - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 with: From 372625e1d0633338c6d2ae02dae15f754a528362 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 24 Sep 2024 08:44:10 -0500 Subject: [PATCH 22/58] style: Move prettier and editorconfig to pre-commit https://github.com/nf-core/modules/pull/4554#issuecomment-1845488824 --- .github/workflows/lint.yml | 32 -------------------------------- .pre-commit-config.yaml | 5 +++++ 2 files changed, 5 insertions(+), 32 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index bb20af0f209..42bc7b8ebcf 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -38,38 +38,6 @@ jobs: with: extra_args: "" - prettier: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: Install NodeJS - uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 - with: - node-version: "20" - - - name: Install Prettier - run: npm install -g prettier@3.1.0 - - - name: Run Prettier --check - run: prettier --check . - - editorconfig: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 - with: - node-version: "20" - - - name: Install editorconfig-checker - run: npm install -g editorconfig-checker - - - name: Run ECLint check - run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test) - ################### # nf-core linting # ################### diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d195d3d9044..056285a3ec2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,3 +29,8 @@ repos: files: \.py$ args: [--fix, --exit-non-zero-on-fix, "--select", "I,E1,E4,E7,E9,F,UP,N"] # sort imports and fix (rules taken from nf-core/tools) - id: ruff-format # formatter + - repo: https://github.com/editorconfig-checker/editorconfig-checker.python + rev: "" # pick a git hash / tag to point to + hooks: + - id: editorconfig-checker + alias: ec From 1670c52b19bebed54cd9c973d30348501c7d981f Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 24 Sep 2024 08:45:34 -0500 Subject: [PATCH 23/58] ci: Add note about nf-core lint pre-commit --- .github/workflows/lint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 42bc7b8ebcf..8a0b32aad6f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -41,6 +41,8 @@ jobs: ################### # nf-core linting # ################### + # TODO Move these to pre-commit + # https://github.com/nf-core/tools/pull/3141 nf-core-changes: name: nf-core-changes runs-on: ubuntu-latest From adf66581a9fc63bd71b469b63f81d395dcda47e4 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 24 Sep 2024 08:53:51 -0500 Subject: [PATCH 24/58] chore: Copy over conda skips https://github.com/nf-core/modules/commit/a004c86eea62d4eed15768c0edb3adcc3f2c1424 https://github.com/nf-core/modules/commit/2da71b7a52a7c6609cb8f29d316ed3fd90d0f7cc --- .github/conda_skip.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/conda_skip.yml b/.github/conda_skip.yml index 44e43a740a4..801fb9616fc 100644 --- a/.github/conda_skip.yml +++ b/.github/conda_skip.yml @@ -45,6 +45,14 @@ # path: modules/nf-core/deepcell/mesmer # - profile: conda # path: modules/nf-core/deepvariant +- profile: conda + path: modules/nf-core/deepvariant/callvariants +- profile: conda + path: modules/nf-core/deepvariant/makeexamples +- profile: conda + path: modules/nf-core/deepvariant/postprocessvariants +- profile: conda + path: modules/nf-core/deepvariant/rundeepvariant # - profile: conda # path: modules/nf-core/ensemblvep/vep # - profile: conda @@ -97,6 +105,8 @@ # path: modules/nf-core/sentieon/bwaindex # - profile: conda # path: modules/nf-core/sentieon/bwamem +- profile: conda + path: modules/nf-core/sentieon/datametrics # - profile: conda # path: modules/nf-core/sentieon/dedup # - profile: conda @@ -127,6 +137,8 @@ # path: subworkflows/nf-core/vcf_annotate_ensemblvep # - profile: conda # path: subworkflows/nf-core/bcl_demultiplex +- profile: conda + path: subworkflows/nf-core/deepvariant # - profile: conda # path: subworkflows/nf-core/fastq_align_bamcmp_bwa # - profile: conda From d0caf9e6d0d4433780aa9e2c6e20a2bedf46ee1f Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 30 Sep 2024 15:17:21 -0500 Subject: [PATCH 25/58] ci: only-changed => changed-since --- .github/workflows/nf-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index a6acf4f0283..69bab0b9ae1 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -143,7 +143,7 @@ jobs: --tap=test.tap \ --ci \ --verbose \ - --only-changed \ + --changed-since HEAD^ \ --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ --filter ${{ matrix.filter }} \ --follow-dependencies From 3b97a33d4ff932a3e38662571221ae22a5b23d20 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 14 Oct 2024 13:35:29 -0500 Subject: [PATCH 26/58] Add confirm-pass --- .github/workflows/lint.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8a0b32aad6f..194280319b7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -148,3 +148,21 @@ jobs: - name: Lint module ${{ matrix.tags }} run: nf-core subworkflows lint ${{ matrix.tags }} + + confirm-pass: + runs-on: ubuntu-latest + needs: [nf-core-lint-modules, nf-core-lint-subworkflows] + if: always() + steps: + - name: All tests ok + if: ${{ success() || !contains(needs.*.result, 'failure') }} + run: exit 0 + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: debug-print + if: always() + run: | + echo "toJSON(needs) = ${{ toJSON(needs) }}" + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" From 865787a1abeffab80edc13d0df37569fc60dc515 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 16 Oct 2024 22:39:03 -0500 Subject: [PATCH 27/58] ci: dynamically set shards --- .github/workflows/nf-test.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 69bab0b9ae1..a6a3eb4ce1d 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -26,6 +26,21 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: + get-number-of-shards: + runs-on: ubuntu-latest + outputs: + # Needs to be a json array + shards: ${{ steps.shards.outputs.shards }} + total_shards: ${{ steps.shards.outputs.total_shards }} + steps: + - id: shards + run: | + nftest_output=$(nf-test test --dry-run --changed-since HEAD^ --filter process --follow-dependencies) + number_of_shards=$(echo $nftest_output | grep -o 'Found [0-9]* related test' | tail -1 | awk '{print $2}') + shards_array=$(for shard in $(seq 1 $number_of_shards); do echo $shard; done | tr ' ' '\n' | jq -R . | jq -s .) + echo "shards=${shards_array}" >> $GITHUB_OUTPUT + echo "total_shards=${number_of_shards}" >> $GITHUB_OUTPUT + nf-test: runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} # NOTE I think this is the cleanest way to get them organized @@ -36,17 +51,19 @@ jobs: # ... # workflow | singularity | 3 name: ${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.shard }} + needs: get-number-of-shards + if: ${{ fromJSON(needs.get-number-of-shards.outputs.shards) != fromJSON('["1", "0"]') }} strategy: fail-fast: false matrix: filter: [process, workflow] profile: [conda, docker_self_hosted, singularity] - shard: [1, 2, 3] + shard: ${{ fromJSON(needs.get-number-of-shards.outputs.shards) }} env: NXF_ANSI_LOG: false NXF_VER: "24.04.4" NFTEST_VER: "0.9.0" - TOTAL_SHARDS: 3 + TOTAL_SHARDS: ${{ needs.get-number-of-shards.outputs.total_shards }} SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} From c141a14e4203c6be8402b27d45416503af39c0ec Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 16 Oct 2024 22:44:19 -0500 Subject: [PATCH 28/58] ci: Run 3 process jobs per CI run as an example --- .github/workflows/nf-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index a6a3eb4ce1d..ec4f90fa40d 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -37,6 +37,7 @@ jobs: run: | nftest_output=$(nf-test test --dry-run --changed-since HEAD^ --filter process --follow-dependencies) number_of_shards=$(echo $nftest_output | grep -o 'Found [0-9]* related test' | tail -1 | awk '{print $2}') + three_tests_per_shard=$(echo $(($number_of_shards / 3)) | awk '{print int($1+0.5)}') shards_array=$(for shard in $(seq 1 $number_of_shards); do echo $shard; done | tr ' ' '\n' | jq -R . | jq -s .) echo "shards=${shards_array}" >> $GITHUB_OUTPUT echo "total_shards=${number_of_shards}" >> $GITHUB_OUTPUT From d3555213d8bafa6a4527625dbcf262782490b81f Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 23 Oct 2024 11:39:07 +0200 Subject: [PATCH 29/58] install nf-test --- .github/workflows/nf-test.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index ec4f90fa40d..0fb42d1352a 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -26,6 +26,15 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: + install-nf-test: + runs-on: ubuntu-latest + steps: + - name: Install nf-test + uses: nf-core/setup-nf-test@v1 + with: + version: ${{ env.NFTEST_VER }} + + get-number-of-shards: runs-on: ubuntu-latest outputs: From f42d82a137290022ed692f28b47b601579f684bc Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 23 Oct 2024 11:42:01 +0200 Subject: [PATCH 30/58] fix install nf-test --- .github/workflows/nf-test.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 0fb42d1352a..3b5028f1e09 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -21,9 +21,12 @@ concurrency: cancel-in-progress: true env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NFT_VER: "0.9.0" + NXF_ANSI_LOG: false NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NXF_VER: "24.04.4" jobs: install-nf-test: @@ -32,7 +35,7 @@ jobs: - name: Install nf-test uses: nf-core/setup-nf-test@v1 with: - version: ${{ env.NFTEST_VER }} + version: ${{ env.NFT_VER }} get-number-of-shards: @@ -70,9 +73,6 @@ jobs: profile: [conda, docker_self_hosted, singularity] shard: ${{ fromJSON(needs.get-number-of-shards.outputs.shards) }} env: - NXF_ANSI_LOG: false - NXF_VER: "24.04.4" - NFTEST_VER: "0.9.0" TOTAL_SHARDS: ${{ needs.get-number-of-shards.outputs.total_shards }} SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} @@ -94,11 +94,6 @@ jobs: with: version: ${{ env.NXF_VER }} - - name: Install nf-test - uses: nf-core/setup-nf-test@v1 - with: - version: ${{ env.NFTEST_VER }} - - name: Setup apptainer if: matrix.profile == 'singularity' uses: eWaterCycle/setup-apptainer@main From c99a2fc24a4598a833d2ab4a8469ffa4b2bf5924 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 23 Oct 2024 11:44:53 +0200 Subject: [PATCH 31/58] fix install nf-test --- .github/workflows/nf-test.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 3b5028f1e09..15ba4ef34da 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -29,15 +29,6 @@ env: NXF_VER: "24.04.4" jobs: - install-nf-test: - runs-on: ubuntu-latest - steps: - - name: Install nf-test - uses: nf-core/setup-nf-test@v1 - with: - version: ${{ env.NFT_VER }} - - get-number-of-shards: runs-on: ubuntu-latest outputs: @@ -45,6 +36,11 @@ jobs: shards: ${{ steps.shards.outputs.shards }} total_shards: ${{ steps.shards.outputs.total_shards }} steps: + - name: Install nf-test + uses: nf-core/setup-nf-test@v1 + with: + version: ${{ env.NFT_VER }} + - id: shards run: | nftest_output=$(nf-test test --dry-run --changed-since HEAD^ --filter process --follow-dependencies) @@ -63,7 +59,7 @@ jobs: # process | docker_self_hosted | 1 # ... # workflow | singularity | 3 - name: ${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.shard }} + name: "${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.shard }}" needs: get-number-of-shards if: ${{ fromJSON(needs.get-number-of-shards.outputs.shards) != fromJSON('["1", "0"]') }} strategy: @@ -89,11 +85,17 @@ jobs: with: distribution: "temurin" java-version: "17" + - name: Setup Nextflow uses: nf-core/setup-nextflow@v2 with: version: ${{ env.NXF_VER }} + - name: Install nf-test + uses: nf-core/setup-nf-test@v1 + with: + version: ${{ env.NFT_VER }} + - name: Setup apptainer if: matrix.profile == 'singularity' uses: eWaterCycle/setup-apptainer@main From c3f3fbe6e9c432adff82f8cf42601f17842b4f86 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 14 Nov 2024 09:23:18 -0600 Subject: [PATCH 32/58] ci: Remove dynamic number of shards, and combine process and workflow filters --- .github/workflows/nf-test.yml | 56 +++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 4ab3ce07074..18061e757ef 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -33,26 +33,27 @@ env: NXF_VER: "24.04.4" jobs: - get-number-of-shards: - runs-on: ubuntu-latest - outputs: - # Needs to be a json array - shards: ${{ steps.shards.outputs.shards }} - total_shards: ${{ steps.shards.outputs.total_shards }} - steps: - - name: Install nf-test - uses: nf-core/setup-nf-test@v1 - with: - version: ${{ env.NFT_VER }} - - - id: shards - run: | - nftest_output=$(nf-test test --dry-run --changed-since HEAD^ --filter process --follow-dependencies) - number_of_shards=$(echo $nftest_output | grep -o 'Found [0-9]* related test' | tail -1 | awk '{print $2}') - three_tests_per_shard=$(echo $(($number_of_shards / 3)) | awk '{print int($1+0.5)}') - shards_array=$(for shard in $(seq 1 $number_of_shards); do echo $shard; done | tr ' ' '\n' | jq -R . | jq -s .) - echo "shards=${shards_array}" >> $GITHUB_OUTPUT - echo "total_shards=${number_of_shards}" >> $GITHUB_OUTPUT + # TODO + # get-number-of-shards: + # runs-on: ubuntu-latest + # outputs: + # # Needs to be a json array + # shards: ${{ steps.shards.outputs.shards }} + # total_shards: ${{ steps.shards.outputs.total_shards }} + # steps: + # - name: Install nf-test + # uses: nf-core/setup-nf-test@v1 + # with: + # version: ${{ env.NFT_VER }} + + # - id: shards + # run: | + # nftest_output=$(nf-test test --dry-run --changed-since HEAD^ --filter process --follow-dependencies) + # number_of_shards=$(echo $nftest_output | grep -o 'Found [0-9]* related test' | tail -1 | awk '{print $2}') + # three_tests_per_shard=$(echo $(($number_of_shards / 3)) | awk '{print int($1+0.5)}') + # shards_array=$(for shard in $(seq 1 $number_of_shards); do echo $shard; done | tr ' ' '\n' | jq -R . | jq -s .) + # echo "shards=${shards_array}" >> $GITHUB_OUTPUT + # echo "total_shards=${number_of_shards}" >> $GITHUB_OUTPUT nf-test: runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} @@ -64,16 +65,19 @@ jobs: # ... # workflow | singularity | 3 name: "${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.shard }}" - needs: get-number-of-shards - if: ${{ fromJSON(needs.get-number-of-shards.outputs.shards) != fromJSON('["1", "0"]') }} + # TODO + # needs: get-number-of-shards + # if: ${{ fromJSON(needs.get-number-of-shards.outputs.shards) != fromJSON('["1", "0"]') }} strategy: fail-fast: false matrix: - filter: [process, workflow] + # NOTE We could split these, but there's probably going to be more process tests than workflow tests, so we're just going to combine them all and bump up the shards for now + # filter: [process, workflow] profile: [conda, docker_self_hosted, singularity] - shard: ${{ fromJSON(needs.get-number-of-shards.outputs.shards) }} + shard: [1, 2, 3, 4, 5] env: - TOTAL_SHARDS: ${{ needs.get-number-of-shards.outputs.total_shards }} + # FIXME Bumping them up to make the transition smooth, then we can throttle them back + TOTAL_SHARDS: 5 SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} @@ -173,7 +177,7 @@ jobs: --verbose \ --changed-since HEAD^ \ --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ - --filter ${{ matrix.filter }} \ + --filter process,workflow \ --follow-dependencies # TODO If matrix == conda create a conda-fail.yml and commit it From 1095134e279b03fa4a97ee941322b525a87de70d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 14 Nov 2024 09:28:25 -0600 Subject: [PATCH 33/58] ci: Remove variable Nextflow versions --- .github/workflows/nf-test.yml | 4 +--- .github/workflows/pytest-workflow.yml | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 18061e757ef..0f44d3f743c 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -26,7 +26,7 @@ concurrency: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - NFT_VER: "0.9.0" + NFT_VER: "0.9.2" NXF_ANSI_LOG: false NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity @@ -96,8 +96,6 @@ jobs: - name: Setup Nextflow uses: nf-core/setup-nextflow@v2 - with: - version: ${{ env.NXF_VER }} - name: Install nf-test uses: nf-core/setup-nf-test@v1 diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 96341063adf..030a4e24886 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -263,10 +263,9 @@ jobs: with: distribution: "temurin" java-version: "17" - - name: Setup Nextflow ${{ matrix.NXF_VER }} + + - name: Setup Nextflow uses: nf-core/setup-nextflow@v2 - with: - version: "${{ matrix.NXF_VER }}" - name: Setup apptainer if: matrix.profile == 'singularity' From b89a08d8b7246d64b9535a5b20a62e9052a9f9f5 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 14 Nov 2024 10:36:52 -0600 Subject: [PATCH 34/58] test: Update bowtie versions --- modules/nf-core/bowtie/build/tests/main.nf.test.snap | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/bowtie/build/tests/main.nf.test.snap b/modules/nf-core/bowtie/build/tests/main.nf.test.snap index 6951ac5b5d4..dde4b29f733 100644 --- a/modules/nf-core/bowtie/build/tests/main.nf.test.snap +++ b/modules/nf-core/bowtie/build/tests/main.nf.test.snap @@ -50,7 +50,7 @@ "content": [ { "BOWTIE_BUILD": { - "bowtie": "1.3.0" + "bowtie": "1.3.1" } } ], @@ -107,4 +107,4 @@ }, "timestamp": "2024-06-18T08:37:53.65689025" } -} \ No newline at end of file +} From 7216e82e13895b391fdfca8fdb902d6e19d6955e Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 14 Nov 2024 10:40:22 -0600 Subject: [PATCH 35/58] ci: Fix indention --- .github/workflows/pytest-workflow.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 030a4e24886..d46e170c41c 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -71,7 +71,6 @@ jobs: tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"] profile: [conda, docker, singularity] exclude: - exclude: - tags: nf-test - profile: conda tags: backsub @@ -239,8 +238,7 @@ jobs: tags: vt/decompose env: NXF_ANSI_LOG: false - - steps: + steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Set up Python From b957e0f5d92156e4e3f6f8deaffd259c5a367c15 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 14 Nov 2024 10:41:02 -0600 Subject: [PATCH 36/58] ci: We're not testing multiple python versions --- .github/workflows/pytest-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index d46e170c41c..ecf17a33383 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -280,7 +280,7 @@ jobs: with: miniconda-version: "latest" channels: conda-forge,bioconda - python-version: ${{ matrix.python-version }} + python-version: "3.11" - name: Conda setup run: | From 350a4cb72dce568d79ef9ddeb9162aaba14b2cd9 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 14 Nov 2024 10:42:41 -0600 Subject: [PATCH 37/58] style: Clean up job names --- .github/workflows/nf-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 0f44d3f743c..8c0f64375ad 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -64,7 +64,7 @@ jobs: # process | docker_self_hosted | 1 # ... # workflow | singularity | 3 - name: "${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.shard }}" + name: "${{ matrix.profile }} | ${{ matrix.shard }}" # TODO # needs: get-number-of-shards # if: ${{ fromJSON(needs.get-number-of-shards.outputs.shards) != fromJSON('["1", "0"]') }} @@ -72,6 +72,7 @@ jobs: fail-fast: false matrix: # NOTE We could split these, but there's probably going to be more process tests than workflow tests, so we're just going to combine them all and bump up the shards for now + # NOTE The name of the test would be name: "${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.shard }}" # filter: [process, workflow] profile: [conda, docker_self_hosted, singularity] shard: [1, 2, 3, 4, 5] From 47e4696961654b7438eb9b8dde9767236e594130 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 14 Nov 2024 10:56:06 -0600 Subject: [PATCH 38/58] build: Remove invisible characters? --- modules/nf-core/bowtie/align/main.nf | 2 +- modules/nf-core/bowtie/build/main.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index aee8821c1d7..e00acce210f 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -5,7 +5,7 @@ process BOWTIE_ALIGN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c8/c8c0819a9b1f520c49c933e667ae50de2a0730ece4c8b9efe79ac5e403963a9f/data' : - 'community​.wave​.seqera​.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" + 'community.wave.seqera.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" input: tuple val(meta), path(reads) diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf index 0f6b9d4d1bf..953e12f6875 100644 --- a/modules/nf-core/bowtie/build/main.nf +++ b/modules/nf-core/bowtie/build/main.nf @@ -5,7 +5,7 @@ process BOWTIE_BUILD { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c8/c8c0819a9b1f520c49c933e667ae50de2a0730ece4c8b9efe79ac5e403963a9f/data' : - 'community​.wave​.seqera​.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" + 'community.wave.seqera.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" input: tuple val(meta), path(fasta) From fad188b4e2f4b6954cf564f33c6de10f1307d0d8 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 14 Nov 2024 10:56:38 -0600 Subject: [PATCH 39/58] test(bowtie): Bump snapshots --- .../bowtie/align/tests/main.nf.test.snap | 36 +++++++++---------- .../bowtie/build/tests/main.nf.test.snap | 22 ++++++------ 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/modules/nf-core/bowtie/align/tests/main.nf.test.snap b/modules/nf-core/bowtie/align/tests/main.nf.test.snap index de95bb81531..e24eff79e62 100644 --- a/modules/nf-core/bowtie/align/tests/main.nf.test.snap +++ b/modules/nf-core/bowtie/align/tests/main.nf.test.snap @@ -2,7 +2,7 @@ "sarscov2 - single_end": { "content": [ [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ], [ "7bdcfc6f54ae6e8f4570395cc85db9a3" @@ -27,10 +27,10 @@ ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-26T09:25:24.60746041" + "timestamp": "2024-11-14T10:55:41.588212" }, "sarscov2 - single_end - stub": { "content": [ @@ -63,7 +63,7 @@ ] ], "3": [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ], "bam": [ [ @@ -93,20 +93,20 @@ ] ], "versions": [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-25T10:00:28.666281812" + "timestamp": "2024-11-14T10:55:46.311312" }, "sarscov2 - paired_end": { "content": [ [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ], [ [ @@ -125,10 +125,10 @@ ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-26T11:57:56.604464368" + "timestamp": "2024-11-14T10:55:51.790156" }, "sarscov2 - paired_end - stub": { "content": [ @@ -155,7 +155,7 @@ ], "3": [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ], "bam": [ [ @@ -179,14 +179,14 @@ ] ], "versions": [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-25T10:01:02.043164876" + "timestamp": "2024-11-14T10:55:56.594761" } } \ No newline at end of file diff --git a/modules/nf-core/bowtie/build/tests/main.nf.test.snap b/modules/nf-core/bowtie/build/tests/main.nf.test.snap index dde4b29f733..e71a80d922f 100644 --- a/modules/nf-core/bowtie/build/tests/main.nf.test.snap +++ b/modules/nf-core/bowtie/build/tests/main.nf.test.snap @@ -18,7 +18,7 @@ ] ], "1": [ - "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + "versions.yml:md5,01c0299b236b1052f80f4be6aacfae6b" ], "index": [ [ @@ -36,15 +36,15 @@ ] ], "versions": [ - "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + "versions.yml:md5,01c0299b236b1052f80f4be6aacfae6b" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-18T08:38:14.852528155" + "timestamp": "2024-11-14T10:56:03.634167" }, "versions": { "content": [ @@ -79,7 +79,7 @@ ] ], "1": [ - "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + "versions.yml:md5,01c0299b236b1052f80f4be6aacfae6b" ], "index": [ [ @@ -97,14 +97,14 @@ ] ], "versions": [ - "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + "versions.yml:md5,01c0299b236b1052f80f4be6aacfae6b" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-18T08:37:53.65689025" + "timestamp": "2024-11-14T10:56:00.181902" } -} +} \ No newline at end of file From 5c48edb27995656e71676dd7f48cff9b3d833fe5 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 15 Nov 2024 11:51:40 -0600 Subject: [PATCH 40/58] style: Remove stray comment --- .github/workflows/lint.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 02ea676ab41..f9480ef9e86 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -36,9 +36,6 @@ jobs: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5 - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 - # FIXME Flip this off once we get to less than a couple hundred. Adding - # this so it will only run against changed files. It'll make it much - # easier to fix these as they come up rather than everything at once. with: extra_args: "" From aba6203a853068bdeaedf99960c39edce5ae7ca5 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 17 Nov 2024 12:56:09 -0600 Subject: [PATCH 41/58] ci: Add GPU tests in their own workflows These will only get trigger on the paths listed --- .github/actions/nf-test-action/action.yml | 6 +- .github/workflows/gpu-tests.yml | 73 +++++++++++++++++++++++ .github/workflows/nf-test.yml | 33 ---------- 3 files changed, 78 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/gpu-tests.yml diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index 06e49ad136a..3b3321e7b8b 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -1,6 +1,9 @@ name: "nf-test Action" description: "Runs nf-test with common setup steps" inputs: + path: + description: "Path to test" + required: false profile: description: "Profile to use" required: true @@ -80,7 +83,8 @@ runs: --changed-since HEAD^ \ --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \ --filter process,workflow \ - --follow-dependencies + --follow-dependencies \ + ${{ inputs.path }} # TODO If matrix == conda create a conda-fail.yml and commit it diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml new file mode 100644 index 00000000000..f59bd407792 --- /dev/null +++ b/.github/workflows/gpu-tests.yml @@ -0,0 +1,73 @@ +name: Run GPU nf-tests +on: + push: + branches: + # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging + - "renovate/**" # branches Renovate creates + pull_request: + branches: [master] + paths: + - "modules/nf-core/parabricks/applybqsr/**" + - "modules/nf-core/parabricks/fq2bam/**" + - "modules/nf-core/parabricks/fq2bammeth/**" + merge_group: + types: [checks_requested] + branches: [master] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "self-hosted" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NFT_VER: "0.9.2" + NXF_ANSI_LOG: false + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + NXF_VER: "24.04.4" + +jobs: + nf-test-gpu: + runs-on: "gpu" + name: "GPU | ${{ matrix.profile }} | ${{ matrix.shard }}" + strategy: + fail-fast: false + matrix: + shard: [1, 2] + include: + - path: modules/nf-core/parabricks/applybqsr + profile: [docker_self_hosted, singularity] + - path: modules/nf-core/parabricks/fq2bam + profile: [docker_self_hosted, singularity] + - path: modules/nf-core/parabricks/fq2bammeth + profile: [docker_self_hosted, singularity] + env: + NXF_ANSI_LOG: false + TOTAL_SHARDS: 2 + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + - name: Run nf-test Action + uses: ./.github/actions/nf-test-action + env: + SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} + SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} + SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} + SENTIEON_AUTH_MECH: "GitHub Actions - token" + with: + path: ${{ matrix.path }} + profile: ${{ matrix.profile }},gpu + shard: ${{ matrix.shard }} + total_shards: ${{ env.TOTAL_SHARDS }} + # TODO Pass a "gpu" tag in case there's cpu only tests diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 6c1b8e5edbc..6270f3a6bba 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -89,36 +89,3 @@ jobs: profile: ${{ matrix.profile }} shard: ${{ matrix.shard }} total_shards: ${{ env.TOTAL_SHARDS }} - - # TODO Reintegrate this somehow - # nf-test-gpu: - # runs-on: "gpu" - # name: nf-test-gpu - # needs: [nf-test-changes] - # if: ( needs.nf-test-changes.outputs.paths != '[]' && contains(needs.nf-test-changes.outputs.paths, 'modules/nf-core/parabricks') ) - # strategy: - # fail-fast: false - # matrix: - # include: - # - path: modules/nf-core/parabricks/applybqsr - # profile: [docker_self_hosted, singularity] - # - path: modules/nf-core/parabricks/fq2bam - # profile: [docker_self_hosted, singularity] - # - path: modules/nf-core/parabricks/fq2bammeth - # profile: [docker_self_hosted, singularity] - # env: - # NXF_ANSI_LOG: false - - # steps: - # - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - # - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - # - name: Run nf-test Action - # uses: ./.github/actions/nf-test-action - # env: - # SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} - # SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} - # SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} - # SENTIEON_AUTH_MECH: "GitHub Actions - token" - # with: - # path: ${{ matrix.path }} - # profile: ${{ matrix.profile }},gpu From 83294657632656b54cb55ee02c885e45f8f04736 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 17 Nov 2024 12:58:45 -0600 Subject: [PATCH 42/58] ci(gpu): Clean up triggers and make a note about how to make the includes smarter --- .github/workflows/gpu-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml index f59bd407792..5d91fa1becc 100644 --- a/.github/workflows/gpu-tests.yml +++ b/.github/workflows/gpu-tests.yml @@ -7,9 +7,8 @@ on: pull_request: branches: [master] paths: - - "modules/nf-core/parabricks/applybqsr/**" - - "modules/nf-core/parabricks/fq2bam/**" - - "modules/nf-core/parabricks/fq2bammeth/**" + - ".github/workflows/gpu-tests.yml" + - "modules/nf-core/parabricks/**" merge_group: types: [checks_requested] branches: [master] @@ -44,6 +43,7 @@ jobs: fail-fast: false matrix: shard: [1, 2] + # TODO Pass these in from GitHub PR trigger events include: - path: modules/nf-core/parabricks/applybqsr profile: [docker_self_hosted, singularity] From 54391d320dad626b56c04f10d9dd25c433bcc4a5 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 17 Nov 2024 16:15:04 -0600 Subject: [PATCH 43/58] ci: pip install cryptography --- .github/actions/nf-test-action/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index 3b3321e7b8b..f1a9d1fc2c3 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -62,6 +62,7 @@ runs: if: env.SENTIEON_ENCRYPTION_KEY != '' && env.SENTIEON_LICENSE_MESSAGE != '' shell: bash run: | + python -m pip install cryptography nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") # TODO If conda-fail.yml exists and matrix = conda skip @@ -69,8 +70,6 @@ runs: - name: Run nf-test shell: bash env: - NFT_DIFF: "pdiff" - NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }} SENTIEON_AUTH_MECH: "GitHub Actions - token" run: | From b477ac34e796eccd914a69aa245407e20b351d7a Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 17 Nov 2024 16:39:46 -0600 Subject: [PATCH 44/58] ci: Try removing path --- .github/actions/nf-test-action/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index f1a9d1fc2c3..4d05d8f5284 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -82,8 +82,7 @@ runs: --changed-since HEAD^ \ --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \ --filter process,workflow \ - --follow-dependencies \ - ${{ inputs.path }} + --follow-dependencies # TODO If matrix == conda create a conda-fail.yml and commit it From c47e4073d44b445a3c1dbad448a7b867f26fe848 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 17 Nov 2024 19:55:34 -0600 Subject: [PATCH 45/58] ci: Add fetch-depth and remove duplicate checkout --- .github/actions/nf-test-action/action.yml | 2 -- .github/workflows/gpu-tests.yml | 3 ++- .github/workflows/nf-test.yml | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index 4d05d8f5284..e01a39f9deb 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -17,8 +17,6 @@ inputs: runs: using: "composite" steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 with: distribution: "temurin" diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml index 5d91fa1becc..9c7e02483b3 100644 --- a/.github/workflows/gpu-tests.yml +++ b/.github/workflows/gpu-tests.yml @@ -57,7 +57,8 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + fetch-depth: 0 - name: Run nf-test Action uses: ./.github/actions/nf-test-action env: diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 6270f3a6bba..96e35e21dc2 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -82,7 +82,8 @@ jobs: SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + fetch-depth: 0 - name: Run nf-test Action uses: ./.github/actions/nf-test-action with: From e04a7430499e5b2e1d31a444e0823fad3e4a9ed1 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 17 Nov 2024 20:41:44 -0600 Subject: [PATCH 46/58] test: Remove the second gpu profile --- tests/config/nf-test.config | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index f651ff4d89d..6a534a8f7c7 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -32,12 +32,6 @@ profiles { docker.enabled = true docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } - gpu { - docker.runOptions = '-u $(id -u):$(id -g) --gpus device=0' - apptainer.runOptions = '--no-mount tmp --writable-tmpfs --nv' - singularity.runOptions = '--no-mount tmp --writable-tmpfs --nv' - use_gpu = true - } docker_self_hosted { docker.enabled = true docker.fixOwnership = true From c7984e5be6569234e38083ace7fa010a97b811a3 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 17 Nov 2024 20:43:13 -0600 Subject: [PATCH 47/58] ci: Add path back in --- .github/actions/nf-test-action/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index e01a39f9deb..18c4b8cffaa 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -80,7 +80,8 @@ runs: --changed-since HEAD^ \ --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \ --filter process,workflow \ - --follow-dependencies + --follow-dependencies \ + ${{ inputs.path }} # TODO If matrix == conda create a conda-fail.yml and commit it From fc7247ab5a7c49436bffa55e4ecfd2936358c3b6 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 18 Nov 2024 09:56:49 -0600 Subject: [PATCH 48/58] ci(gpu): Remove changed-since --- .github/actions/nf-test-action/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index 18c4b8cffaa..87a94dbad15 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -77,7 +77,7 @@ runs: --tap=test.tap \ --verbose \ --ci \ - --changed-since HEAD^ \ + # FIXME this doesn't work with a path? --changed-since HEAD^ \ --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \ --filter process,workflow \ --follow-dependencies \ From ec92b74e40a72a81ae5e36356c5bafa4ff4ca8b7 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 18 Nov 2024 10:11:04 -0600 Subject: [PATCH 49/58] ci: Fix missed dorny/paths-filter update Co-authored-by: mashehu --- .github/workflows/lint.yml | 3 +-- .github/workflows/pytest-workflow.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f9480ef9e86..039e5069a7f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -56,8 +56,7 @@ jobs: with: fetch-depth: 2 # To retrieve the preceding commit. - # TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented - - uses: mirpedrol/paths-filter@main + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 id: filter with: filters: "tests/config/pytest_modules.yml" diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index ecf17a33383..a772dfc1db1 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -42,8 +42,7 @@ jobs: with: fetch-depth: 2 # To retrieve the preceding commit. - # TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented - - uses: mirpedrol/paths-filter@main + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 id: filter with: filters: "tests/config/pytest_modules.yml" From 3f5fe11896e1e08f1a1b0e8fe893944c7035a168 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 18 Nov 2024 11:50:50 -0600 Subject: [PATCH 50/58] Add changes from codereview Co-authored-by: mashehu --- .github/workflows/nf-test.yml | 21 --------------------- .pre-commit-config.yaml | 2 +- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 96e35e21dc2..38ebea1eff9 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -33,27 +33,6 @@ env: NXF_VER: "24.04.4" jobs: - # TODO - # get-number-of-shards: - # runs-on: ubuntu-latest - # outputs: - # # Needs to be a json array - # shards: ${{ steps.shards.outputs.shards }} - # total_shards: ${{ steps.shards.outputs.total_shards }} - # steps: - # - name: Install nf-test - # uses: nf-core/setup-nf-test@v1 - # with: - # version: ${{ env.NFT_VER }} - - # - id: shards - # run: | - # nftest_output=$(nf-test test --dry-run --changed-since HEAD^ --filter process --follow-dependencies) - # number_of_shards=$(echo $nftest_output | grep -o 'Found [0-9]* related test' | tail -1 | awk '{print $2}') - # three_tests_per_shard=$(echo $(($number_of_shards / 3)) | awk '{print int($1+0.5)}') - # shards_array=$(for shard in $(seq 1 $number_of_shards); do echo $shard; done | tr ' ' '\n' | jq -R . | jq -s .) - # echo "shards=${shards_array}" >> $GITHUB_OUTPUT - # echo "total_shards=${number_of_shards}" >> $GITHUB_OUTPUT nf-test: runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} # NOTE I think this is the cleanest way to get them organized diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f6e6fa5a3bb..b6517edd742 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: args: [--fix, --exit-non-zero-on-fix, "--select", "I,E1,E4,E7,E9,F,UP,N"] # sort imports and fix (rules taken from nf-core/tools) - id: ruff-format # formatter - repo: https://github.com/editorconfig-checker/editorconfig-checker.python - rev: "" # pick a git hash / tag to point to + rev: "3.0.3" hooks: - id: editorconfig-checker alias: ec From 9d05040b526b0ea7ff249165bdea5df1b92967c3 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 18 Nov 2024 11:59:40 -0600 Subject: [PATCH 51/58] style: Remove example TODO comments https://github.com/nf-core/modules/pull/6286/files#r1846856369 Co-authored-by: mashehu --- .github/actions/nf-test-action/action.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index 87a94dbad15..5e30fed7b89 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -63,8 +63,8 @@ runs: python -m pip install cryptography nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") - # TODO If conda-fail.yml exists and matrix = conda skip - # Really we need in-verse tag selection https://github.com/askimed/nf-test/issues/260 + # TODO Skip failing conda tests and document their failures + # https://github.com/nf-core/modules/issues/7017 - name: Run nf-test shell: bash env: @@ -83,8 +83,6 @@ runs: --follow-dependencies \ ${{ inputs.path }} - # TODO If matrix == conda create a conda-fail.yml and commit it - # TODO If no test.tap, then make one to spoof? - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 if: ${{ inputs.path != '' }} From e04d248e5888a3e6d5a5445d6ceeec9eb8767632 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 18 Nov 2024 12:11:07 -0600 Subject: [PATCH 52/58] refactor: Try tags for GPU CI https://github.com/askimed/nf-test/issues/272 --- .github/actions/nf-test-action/action.yml | 11 ++++++----- .github/workflows/gpu-tests.yml | 9 ++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index 5e30fed7b89..a656fadbb00 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -1,9 +1,6 @@ name: "nf-test Action" description: "Runs nf-test with common setup steps" inputs: - path: - description: "Path to test" - required: false profile: description: "Profile to use" required: true @@ -13,6 +10,9 @@ inputs: total_shards: description: "Total number of test shards(NOT the total number of matrix jobs)" required: true + tags: + description: "Tags to test (`[,...]`)" + required: false runs: using: "composite" @@ -70,6 +70,7 @@ runs: env: SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }} SENTIEON_AUTH_MECH: "GitHub Actions - token" + TAGS: ${{ inputs.tags && format('--tag {0}', inputs.tags) || '' }} run: | NFT_WORKDIR=~ \ nf-test test \ @@ -77,11 +78,11 @@ runs: --tap=test.tap \ --verbose \ --ci \ - # FIXME this doesn't work with a path? --changed-since HEAD^ \ + --changed-since HEAD^ \ --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \ --filter process,workflow \ --follow-dependencies \ - ${{ inputs.path }} + ${{ env.TAGS }} # TODO If no test.tap, then make one to spoof? - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml index 9c7e02483b3..a6634f9047e 100644 --- a/.github/workflows/gpu-tests.yml +++ b/.github/workflows/gpu-tests.yml @@ -45,11 +45,11 @@ jobs: shard: [1, 2] # TODO Pass these in from GitHub PR trigger events include: - - path: modules/nf-core/parabricks/applybqsr + - tags: parabricks/applybqsr profile: [docker_self_hosted, singularity] - - path: modules/nf-core/parabricks/fq2bam + - tags: parabricks/fq2bam profile: [docker_self_hosted, singularity] - - path: modules/nf-core/parabricks/fq2bammeth + - tags: parabricks/fq2bammeth profile: [docker_self_hosted, singularity] env: NXF_ANSI_LOG: false @@ -67,8 +67,7 @@ jobs: SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} SENTIEON_AUTH_MECH: "GitHub Actions - token" with: - path: ${{ matrix.path }} profile: ${{ matrix.profile }},gpu shard: ${{ matrix.shard }} total_shards: ${{ env.TOTAL_SHARDS }} - # TODO Pass a "gpu" tag in case there's cpu only tests + tags: ${{matrix.tags}},gpu From 9863eba580a0359a8486f65a34bdf7c94a48cd33 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 18 Nov 2024 12:11:50 -0600 Subject: [PATCH 53/58] style: Set NFT_WORKDIR as an ENV variable --- .github/actions/nf-test-action/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index a656fadbb00..5b87e1e3611 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -71,8 +71,8 @@ runs: SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }} SENTIEON_AUTH_MECH: "GitHub Actions - token" TAGS: ${{ inputs.tags && format('--tag {0}', inputs.tags) || '' }} + NFT_WORKDIR: "~" run: | - NFT_WORKDIR=~ \ nf-test test \ --profile=${{ inputs.profile }} \ --tap=test.tap \ From d134e78a0aa14fef469e30c8d90006092c9f3a6b Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 18 Nov 2024 12:29:00 -0600 Subject: [PATCH 54/58] style: Run lsp format on a parabricks module For science --- modules/nf-core/parabricks/applybqsr/main.nf | 31 ++++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/modules/nf-core/parabricks/applybqsr/main.nf b/modules/nf-core/parabricks/applybqsr/main.nf index 38b2af002b8..0607a11d08a 100644 --- a/modules/nf-core/parabricks/applybqsr/main.nf +++ b/modules/nf-core/parabricks/applybqsr/main.nf @@ -1,8 +1,7 @@ process PARABRICKS_APPLYBQSR { - tag "$meta.id" + tag "${meta.id}" label 'process_high' label 'process_gpu' - container "nvcr.io/nvidia/clara/clara-parabricks:4.3.2-1" input: @@ -15,7 +14,7 @@ process PARABRICKS_APPLYBQSR { output: tuple val(meta), path("*.bam"), emit: bam tuple val(meta), path("*.bai"), emit: bai - path "versions.yml" , emit: versions + path "versions.yml", emit: versions when: task.ext.when == null || task.ext.when @@ -23,28 +22,28 @@ process PARABRICKS_APPLYBQSR { script: // Exit if running this module with -profile conda / -profile mamba if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." + error("Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead.") } def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def interval_command = intervals ? intervals.collect{"--interval-file $it"}.join(' ') : "" - def copy_index_command = bam_index ? "cp -L $bam_index `readlink -f $bam`.bai" : "" - def num_gpus = task.accelerator ? "--num-gpus $task.accelerator.request" : '' + def interval_command = intervals ? intervals.collect { "--interval-file ${it}" }.join(' ') : "" + def copy_index_command = bam_index ? "cp -L ${bam_index} `readlink -f ${bam}`.bai" : "" + def num_gpus = task.accelerator ? "--num-gpus ${task.accelerator.request}" : '' """ # parabricks complains when index is not a regular file in the same directory as the bam # copy the index to this path. - $copy_index_command + ${copy_index_command} pbrun \\ applybqsr \\ - --ref $fasta \\ - --in-bam $bam \\ - --in-recal-file $bqsr_table \\ - $interval_command \\ + --ref ${fasta} \\ + --in-bam ${bam} \\ + --in-recal-file ${bqsr_table} \\ + ${interval_command} \\ --out-bam ${prefix}.bam \\ - --num-threads $task.cpus \\ - $num_gpus \\ - $args + --num-threads ${task.cpus} \\ + ${num_gpus} \\ + ${args} cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -55,7 +54,7 @@ process PARABRICKS_APPLYBQSR { stub: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def interval_command = intervals ? intervals.collect{"--interval-file $it"}.join(' ') : "" + def interval_command = intervals ? intervals.collect { "--interval-file ${it}" }.join(' ') : "" """ touch ${prefix}.bam touch ${prefix}.bam.bai From a5557afb330d34648d60a0d5f63886ee8c3c8289 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 18 Nov 2024 12:56:04 -0600 Subject: [PATCH 55/58] test: Add gpu tags to parabricks --- modules/nf-core/parabricks/applybqsr/tests/main.nf.test | 3 ++- modules/nf-core/parabricks/fq2bam/tests/main.nf.test | 3 ++- modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/nf-core/parabricks/applybqsr/tests/main.nf.test b/modules/nf-core/parabricks/applybqsr/tests/main.nf.test index c85fd447b3d..9cecef89358 100644 --- a/modules/nf-core/parabricks/applybqsr/tests/main.nf.test +++ b/modules/nf-core/parabricks/applybqsr/tests/main.nf.test @@ -8,6 +8,7 @@ nextflow_process { tag "modules_nfcore" tag "parabricks" tag "parabricks/applybqsr" + tag "gpu" test("sarscov2 - bam - pe") { @@ -210,4 +211,4 @@ nextflow_process { } -} +} \ No newline at end of file diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index 8e4adb2966c..c2273e762ba 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -10,6 +10,7 @@ nextflow_process { tag "parabricks/fq2bam" tag "modules_nfcore" tag "parabricks" + tag "gpu" setup { run("BWA_INDEX") { @@ -171,4 +172,4 @@ nextflow_process { } -} +} \ No newline at end of file diff --git a/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test b/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test index 42e436a3ba3..459eb6317db 100644 --- a/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test @@ -10,6 +10,7 @@ nextflow_process { tag "parabricks" tag "parabricks/fq2bammeth" tag "modules_nfcore" + tag "gpu" setup { run("BWAMETH_INDEX") { @@ -119,4 +120,4 @@ nextflow_process { ) } } -} +} \ No newline at end of file From 08a16d4fbb6bccf1904e72fc9a92733380d65980 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 18 Nov 2024 13:35:29 -0600 Subject: [PATCH 56/58] ci: Split tags out of matrix --- .github/workflows/gpu-tests.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml index a6634f9047e..8acc410991b 100644 --- a/.github/workflows/gpu-tests.yml +++ b/.github/workflows/gpu-tests.yml @@ -43,14 +43,12 @@ jobs: fail-fast: false matrix: shard: [1, 2] + profile: [docker_self_hosted, singularity] # conda? # TODO Pass these in from GitHub PR trigger events - include: - - tags: parabricks/applybqsr - profile: [docker_self_hosted, singularity] - - tags: parabricks/fq2bam - profile: [docker_self_hosted, singularity] - - tags: parabricks/fq2bammeth - profile: [docker_self_hosted, singularity] + tags: + - parabricks/applybqsr + - parabricks/fq2bam + - parabricks/fq2bammeth env: NXF_ANSI_LOG: false TOTAL_SHARDS: 2 From ee7ffc0a02c5e79a93ae4bd7a04e9a661602fe6b Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 18 Nov 2024 15:28:54 -0600 Subject: [PATCH 57/58] Revert "style: Run lsp format on a parabricks module" This reverts commit d134e78a0aa14fef469e30c8d90006092c9f3a6b. --- modules/nf-core/parabricks/applybqsr/main.nf | 31 ++++++++++---------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/modules/nf-core/parabricks/applybqsr/main.nf b/modules/nf-core/parabricks/applybqsr/main.nf index 0607a11d08a..38b2af002b8 100644 --- a/modules/nf-core/parabricks/applybqsr/main.nf +++ b/modules/nf-core/parabricks/applybqsr/main.nf @@ -1,7 +1,8 @@ process PARABRICKS_APPLYBQSR { - tag "${meta.id}" + tag "$meta.id" label 'process_high' label 'process_gpu' + container "nvcr.io/nvidia/clara/clara-parabricks:4.3.2-1" input: @@ -14,7 +15,7 @@ process PARABRICKS_APPLYBQSR { output: tuple val(meta), path("*.bam"), emit: bam tuple val(meta), path("*.bai"), emit: bai - path "versions.yml", emit: versions + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -22,28 +23,28 @@ process PARABRICKS_APPLYBQSR { script: // Exit if running this module with -profile conda / -profile mamba if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - error("Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead.") + error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." } def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def interval_command = intervals ? intervals.collect { "--interval-file ${it}" }.join(' ') : "" - def copy_index_command = bam_index ? "cp -L ${bam_index} `readlink -f ${bam}`.bai" : "" - def num_gpus = task.accelerator ? "--num-gpus ${task.accelerator.request}" : '' + def interval_command = intervals ? intervals.collect{"--interval-file $it"}.join(' ') : "" + def copy_index_command = bam_index ? "cp -L $bam_index `readlink -f $bam`.bai" : "" + def num_gpus = task.accelerator ? "--num-gpus $task.accelerator.request" : '' """ # parabricks complains when index is not a regular file in the same directory as the bam # copy the index to this path. - ${copy_index_command} + $copy_index_command pbrun \\ applybqsr \\ - --ref ${fasta} \\ - --in-bam ${bam} \\ - --in-recal-file ${bqsr_table} \\ - ${interval_command} \\ + --ref $fasta \\ + --in-bam $bam \\ + --in-recal-file $bqsr_table \\ + $interval_command \\ --out-bam ${prefix}.bam \\ - --num-threads ${task.cpus} \\ - ${num_gpus} \\ - ${args} + --num-threads $task.cpus \\ + $num_gpus \\ + $args cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -54,7 +55,7 @@ process PARABRICKS_APPLYBQSR { stub: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def interval_command = intervals ? intervals.collect { "--interval-file ${it}" }.join(' ') : "" + def interval_command = intervals ? intervals.collect{"--interval-file $it"}.join(' ') : "" """ touch ${prefix}.bam touch ${prefix}.bam.bai From c0771d648981e4ef5a5a52f06eaa2712e5f57725 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 18 Nov 2024 17:14:58 -0600 Subject: [PATCH 58/58] ci: Fix name --- .github/workflows/gpu-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml index 8acc410991b..dfabddf07ca 100644 --- a/.github/workflows/gpu-tests.yml +++ b/.github/workflows/gpu-tests.yml @@ -38,7 +38,7 @@ env: jobs: nf-test-gpu: runs-on: "gpu" - name: "GPU | ${{ matrix.profile }} | ${{ matrix.shard }}" + name: "GPU | ${{ matrix.tags}} | ${{ matrix.profile }} | ${{ matrix.shard }}" strategy: fail-fast: false matrix: