ci(check-fixtures): parallelise fixture checking commands

Introduces several independent `run-tests.sh` fixture-checking commands in order to speed up fixture checking by parallelisation. Renames `run-tests.sh` script options and CI rules to better separate data checks, formatting checks and linting checks. Adds `shfmt` formatting checks, `flake8` and `yamllint` linting checks. Removes `pydocstyle` formating checks since we moved to `black` code formatter. Introduces `/run-tests.sh --help` explaining all the checking options. Updates CI environment to Ubuntu 24.04 and latest actions (`actions/checkout@v4`, `actions/setup-node@v4`, `actions/setup-python@v5`). Amends `.editorconfig` to add rules for shell scripts and remove rules for ReST files that are no longer needed after switch to Markdown. BREAKING CHANGE: Refactors `run-tests.sh` script options.
cernopendata · Jan 10, 2025 · e9c16fc · e9c16fc
1 parent 952b104
commit e9c16fc
Show file tree

Hide file tree

Showing 8 changed files with 288 additions and 129 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of CERN Open Data Portal.
-# Copyright (C) 2016 CERN.
+# Copyright (C) 2016, 2025 CERN.
 #
 # CERN Open Data Portal is free software; you can redistribute it
 # and/or modify it under the terms of the GNU General Public License as
@@ -40,8 +40,8 @@ known_first_party = cernopendata
 multi_line_output = 2
 default_section = THIRDPARTY
 
-# RST files (used by sphinx)
-[*.rst]
+# Shell script files
+[*.sh]
 indent_size = 4
 
 # CSS, HTML, JS, JSON, YML

diff --git a/.flake8 b/.flake8
@@ -0,0 +1,19 @@
+[flake8]
+max-line-length = 89
+
+exclude =
+    build
+    dist
+    docs
+    coverage.xml
+    cernopendata_client.egg-info
+    .*/
+    env/
+    .git
+    __pycache__
+
+ignore = E203, E231, E266, E501, W503, F403, F401, W605
+
+max-complexity = 18
+
+select = B,C,E,F,W,T4,B9
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,5 +1,5 @@
 # This file is part of CERN Open Data Portal.
-# Copyright (C) 2020, 2023, 2024 CERN.
+# Copyright (C) 2020, 2023, 2024, 2025 CERN.
 #
 # CERN Open Data Portal is free software; you can redistribute it
 # and/or modify it under the terms of the GNU General Public License as
@@ -20,117 +20,199 @@
 # waive the privileges and immunities granted to it by virtue of its status
 # as an Intergovernmental Organization or submit itself to any jurisdiction.
 
-name: CI
+name: ci
 
 on: [push, pull_request]
 
 jobs:
-  lint-shellcheck:
-    runs-on: ubuntu-20.04
+  data-dois:
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
-      - name: Runs shell script static analysis
-        run: |
-          sudo apt-get install shellcheck
-          ./run-tests.sh --check-shellscript
+      - name: Setup node
+        uses: actions/setup-node@v4
 
-  lint-black:
-    runs-on: ubuntu-20.04
+      - name: Check data DOIs
+        run: ./run-tests.sh --data-dois
+
+  data-licenses:
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
 
-      - name: Check Python code formatting
+      - name: Check data licenses
         run: |
-          pip install black
-          ./run-tests.sh --check-black
+          ./run-tests.sh --data-licenses
 
-  lint-pycodestyle:
-    runs-on: ubuntu-20.04
+  data-recids:
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+
+      - name: Check data record IDs
+        run: ./run-tests.sh --data-recids
+
+  data-slugs:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+
+      - name: Check data slugs
+        run: ./run-tests.sh --data-slugs
+
+  data-types:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+
+      - name: Check data types
+        run: ./run-tests.sh --data-types
+
+  format-black:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
 
-      - name: Check compliance with Python coding style conventions
+      - name: Check formatting of Python code
         run: |
-          pip install --upgrade pip
-          pip install pycodestyle
-          ./run-tests.sh --check-pycodestyle
+          pip install black
+          ./run-tests.sh --format-black
 
-  lint-pydocstyle:
-    runs-on: ubuntu-20.04
+  format-isort:
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
 
-      - name: Check compliance with Python docstring conventions
+      - name: Check formatting of Python imports
         run: |
-          pip install --upgrade pip
-          pip install pydocstyle
-          ./run-tests.sh --check-pydocstyle
+          pip install isort
+          ./run-tests.sh --format-isort
 
-  check-fixtures:
-    runs-on: ubuntu-20.04
+  format-shfmt:
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+
+      - name: Check formatting of shell scripts
+        run: |
+          sudo apt-get install shfmt
+          ./run-tests.sh --format-shfmt
+
+  format-whitespace:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
 
       - name: Setup node
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v4
+
+      - name: Check formatting of whitespace
+        run: ./run-tests.sh --format-whitespace
+
+  lint-flake8:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
         with:
-          node-version: "14"
+          python-version: 3.9
 
-      - name: Install Node dependencies
-        run: npm install -g jsonlint
+      - name: Check linting of Python code
+        run: |
+          pip install flake8
+          ./run-tests.sh --lint-flake8
+
+  lint-jsonlint:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
 
-      - name: Check fixtures
-        run: ./run-tests.sh --check-fixtures
+      - name: Setup Node
+        uses: actions/setup-node@v4
 
-  check-isort:
-    runs-on: ubuntu-20.04
+      - name: Check linting of JSON files
+        run: |
+          npm install jsonlint --global
+          ./run-tests.sh --lint-jsonlint
+
+  lint-pydocstyle:
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
 
-      - name: Check isort
+      - name: Check linting of Python docstrings
         run: |
-          pip install --upgrade pip
-          pip install isort
-          ./run-tests.sh --check-isort
+          pip install pydocstyle
+          ./run-tests.sh --lint-pydocstyle
+
+  lint-shellcheck:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
 
-  check-licenses:
-    runs-on: ubuntu-20.04
+      - name: Check linting of shell scripts
+        run: |
+          sudo apt-get install shellcheck
+          ./run-tests.sh --lint-shellcheck
+
+  lint-yamllint:
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: "3.12"
 
-      - name: Check licenses
+      - name: Lint YAML files
         run: |
-          pip install --upgrade pip
-          ./run-tests.sh --check-licenses
+          pip install yamllint
+          ./run-tests.sh --lint-yamllint
diff --git a/.yamllint.yaml b/.yamllint.yaml
@@ -0,0 +1,8 @@
+extends: default
+
+rules:
+  comments:
+    min-spaces-from-content: 1
+  document-start: disable
+  line-length: disable
+  truthy: disable
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,4 +1,4 @@
 include:
- - path:
-   - ../cernopendata-portal/docker-compose.yml
-   - docker-compose-override.yml
+  - path:
+      - ../cernopendata-portal/docker-compose.yml
+      - docker-compose-override.yml