From eea6737fc295974a441c55be6c103346158d073a Mon Sep 17 00:00:00 2001 From: Azriel Fasten Date: Wed, 24 Apr 2024 21:02:33 +0300 Subject: [PATCH] Add changeguard. * Had to make all the README outputs reproducible. --- .github/workflows/build-and-test.yml | 13 +- .gitignore | 4 +- .pre-commit-config.yaml | 1 + README.example.output.svg | 21 + README.example.terminal.svg | 292 ++++++++++++++ README.help.generated.svg | 584 ++++++++++++++++----------- README.md | 8 +- README.md.jinja2 | 12 +- README.simple.sh.generated.svg | 369 ----------------- examples/simple.sh | 4 +- package-lock.json | 51 +-- scripts/generate-readme.sh | 24 +- scripts/pre.sh | 40 +- scripts/pyproject.toml | 29 +- scripts/run-all-examples.sh | 8 + scripts/run-ood-smoke-test.sh | 2 + scripts/utilities/changeguard.sh | 59 +++ scripts/utilities/pin-extra-reqs.py | 114 ++++++ scripts/utilities/pin-extra-reqs.sh | 99 +++++ 19 files changed, 1080 insertions(+), 654 deletions(-) create mode 100644 README.example.output.svg create mode 100644 README.example.terminal.svg delete mode 100644 README.simple.sh.generated.svg create mode 100755 scripts/utilities/changeguard.sh create mode 100644 scripts/utilities/pin-extra-reqs.py create mode 100755 scripts/utilities/pin-extra-reqs.sh diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index a3a3a1e..a397da7 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -27,8 +27,15 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.8.0 - - name: Install Bash (Ubuntu) - run: sudo apt-get update && sudo apt-get install -y bash grep xxd git xxhash rsync + - name: Install Bash and other dependencies (Ubuntu) + # * bash: scripts. + # * xxd: tests. + # * git: scripts, tests. + # * xxhash: changeguard. + # * rsync: out-of-directory test. + # * grep: tests. + # * expect: for `unbuffer`, useful to grab and compare ansi color symbols. + run: sudo apt-get update && sudo apt-get install -y bash grep xxd git xxhash rsync expect # - name: Install nvm # run: | # curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash @@ -36,5 +43,5 @@ jobs: run: | npm install npx playwright install-deps - npx playwright install + npx playwright install firefox bash scripts/pre.sh diff --git a/.gitignore b/.gitignore index 8a93bda..6b0b2a3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ NOTES.md jsconfig.json -.cache/ -node_modules/ +.cache +node_modules screenshots/ .deleteme/ *.egg-info/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 53f87cb..10acf93 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,6 +24,7 @@ repos: - id: name-tests-test - id: requirements-txt-fixer - id: trailing-whitespace + exclude: ^.*\.(svg|log)$ - id: check-byte-order-marker - repo: https://github.com/jlebar/pre-commit-hooks rev: f2d115a052860b09b2888b4f104be614bf3b4779 diff --git a/README.example.output.svg b/README.example.output.svg new file mode 100644 index 0000000..c70feb8 --- /dev/null +++ b/README.example.output.svg @@ -0,0 +1,21 @@ + + + + + + + + Polygon MeshHeightmap2D Implicit Surface3D Implicit SurfaceVoxel GridMesh GenerationPolygon Rendering PipelineStartRegular SamplingVoxelization(Regular Sampling)Regular SamplingExample: SDFieldRegular SamplingExample: SDFieldIsosurface ExtractionAdaptive Voxel GridAdaptive Sampling(Feature Preservation)Ray Marching/Casting3D MipmapSame data, differentstructureExample: SVOExample: GigaVoxelsAuxiliary Data Structures: Puzzle Pieces2D Height MipmapDownsampling (LOD)Adaptive HeightmapDownsampling(LOD)MeshGenerationExample: GigaVoxelsExample: MinecraftExample: Clipmaps{Toroidal} {Chunked} {Adaptive} SplattingBVHRay Tracing PipelineShear-Warp Volume Rendering{Streaming} Fourier VolumeFFTFourier Volume RenderingExample: BrickMapDownsampling(LOD)Example: SDFieldIsosurface ExtractionSame data, differentstructureExample: Transvoxel Algorithmx \ No newline at end of file diff --git a/README.example.terminal.svg b/README.example.terminal.svg new file mode 100644 index 0000000..ce7b04a --- /dev/null +++ b/README.example.terminal.svg @@ -0,0 +1,292 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +$ npx --no-install excalidraw-brute-export-cli -i . +/examples/ +simple.excalidraw + + +--background  +1 + --embed-scene  +0 + --dark-mode  +0 + --scale  +1 + --format svg -o  + + +"./README.example.output.svg" + --verbose + + +console.log('amain') + + +info +: logger.info('amain') + + +info +: options: + + +info +: { + + +  i:  +'./examples/simple.excalidraw' +, + + +  input:  +'./examples/simple.excalidraw' +, + + +  background:  +true +, + + +  b:  +true +, + + +  embedScene:  +false +, + + +  e:  +false +, + + +  darkMode:  +false +, + + +  d:  +false +, + + +  scale:  +1 +, + + +  s:  +1 +, + + +  format:  +'svg' +, + + +  f:  +'svg' +, + + +  o:  +'./README.example.output.svg' +, + + +  output:  +'./README.example.output.svg' +, + + +  verbose:  +true +, + + +  url:  +'https://excalidraw.com/' +, + + +  headless:  +true +, + + +  screenshots:  +'' + + +} + + +info +: OK file is chosen? + + +info +: fileChooserPromise! + + +info +: await exportScaleElements.count(): 3 + + +info +: fileChooserPromise! + + +info +: closing... + + +program.run() done + + + + + \ No newline at end of file diff --git a/README.help.generated.svg b/README.help.generated.svg index 229b608..b4b9926 100644 --- a/README.help.generated.svg +++ b/README.help.generated.svg @@ -1,5 +1,5 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - -$ npx excalidraw-brute-export-cli --help - + +$ npx excalidraw-brute-export-cli --help + + + + +  excalidraw-brute-export-cli 0.2.0 —  +Export Excalidraw using a (headless)  + + +browser via the command line. + + + + +USAGE + + + + +▸ + excalidraw-brute-export-cli   +<OPTIONS...> + + + + + + + + +OPTIONS + + + + + -b +, + --background +<background> +        Should the background be exported, or  + + +should it be      + + +                                         transparent. Same option as in the  + + +Excalidraw UI when   + + +                                         exporting an image.                     + + +boolean + + + -d +, + --dark-mode +<dark-mode> +          Should the diagram be exported in dark  + + +mode, or not.    + + +                                         Same option as in the Excalidraw UI  + + +when exporting an   + + +                                         image.                                  + + +boolean + - + -e +, + --embed-scene +<embed-scene> +      Should the diagram be exported with the + -  excalidraw-brute-export-cli 0.2.0 —  -Export Excalidraw using a (headless) browser via the command line. - +scene embedded  + - +                                         in the image, or not. Tooltip from the  + -USAGE - +UI: 'Scene data  + - +                                         will be saved into the exported PNG/SVG + -▸ - excalidraw-brute-export-cli   -<OPTIONS...> - +file so that    + - +                                         the scene can be restored from it.      + - +                                         Will increase exported file size.' Same + - +option as in    + -OPTIONS - +                                         the Excalidraw UI when exporting an  + - +image.              + - -b -, - --background -<background> -        Should the background be exported, or should it be      - +boolean + -                                         transparent. Same option as in the Excalidraw UI when   - + -f +, + --format +<format> +                The format to export the image in.  + -                                         exporting an image.                                     - +Either "png" or      + -boolean - +required +                             "svg".                                  + - -d -, - --dark-mode -<dark-mode> -          Should the diagram be exported in dark mode, or not.    - +one of "png","svg" + -                                         Same option as in the Excalidraw UI when exporting an   - + --headless +[headless] +                Should the browser be headless.         + -                                         image.                                                  - +boolean, default: true + -boolean - + -i +, + --input +<excalidaraw-path> +       The path to the excalidraw file         + - -e -, - --embed-scene -<embed-scene> -      Should the diagram be exported with the scene embedded  - +required + -                                         in the image, or not. Tooltip from the UI: 'Scene data  - + -o +, + --output +<output-path> +           The path to the output file.            + -                                         will be saved into the exported PNG/SVG file so that    - +required + -                                         the scene can be restored from it.                      - + -s +, + --scale +<scale> +                  The scale to use when exporting the  + -                                         Will increase exported file size.' Same option as in    - +image. Same option  + -                                         the Excalidraw UI when exporting an image.              - +required +                             as in the Excalidraw UI when exporting  + -boolean - +an image.        + - -f -, - --format -<format> -                The format to export the image in. Either "png" or      - +one of 1,2,3 + -required -                             "svg".                                                  - + --screenshots +[screenshots] +          Path to store debug screenshots at each + -one of "png","svg" - +step. Empty     + - --headless -[headless] -                Should the browser be headless.                         - +                                         string means no screenshots are  + -boolean, default: true - +recored. Defaults to no + - -i -, - --input -<excalidaraw-path> -       The path to the excalidraw file                         - +                                         screenshots.                            + -required - +default: "" + - -o -, - --output -<output-path> -           The path to the output file.                            - + -u +, + --url +[url] +                      The URL to use for excalidraw website.  + -required - +default: "https://excalidraw.com/" + - -s -, - --scale -<scale> -                  The scale to use when exporting the image. Same option  - + -required -                             as in the Excalidraw UI when exporting an image.        - +GLOBAL OPTIONS + -one of 1,2,3 - + - --screenshots -[screenshots] -          Path to store debug screenshots at each step. Empty     - + -h +, + --help +                           Display global help or command-related  + -                                         string means no screenshots are recored. Defaults to no - +help.            + -                                         screenshots.                                            - + -V +, + --version +                        Display version.                        + -default: "" - + --no-color +                           Disable use of colors in output.        + - -u -, - --url -[url] -                      The URL to use for excalidraw website.                  - + -v +, + --verbose +                        Verbose mode: will also output debug  + -default: "https://excalidraw.com/" - +messages.          + - + --quiet +                              Quiet mode - only displays warn and  + -GLOBAL OPTIONS - +error messages.     + - + --silent +                             Silent mode: does not output anything,  + - -h -, - --help -                           Display global help or command-related help.            - +giving no        + - -V -, - --version -                        Display version.                                        - - - --no-color -                           Disable use of colors in output.                        - - - -v -, - --verbose -                        Verbose mode: will also output debug messages.          - +                                         indication of success or failure other  + - --quiet -                              Quiet mode - only displays warn and error messages.     - +than the exit    + - --silent -                             Silent mode: does not output anything, giving no        - +                                         code.                                   + -                                         indication of success or failure other than the exit    - - -                                         code.                                                   - - - + -program.run() done - +program.run() done + diff --git a/README.md b/README.md index b8b9873..b4b3ac5 100644 --- a/README.md +++ b/README.md @@ -50,12 +50,16 @@ npm install # Might prompt for root. npx playwright install-deps -npx playwright install +npx playwright install firefox ``` Example: -Output of `./examples/simple.sh` +Output of `./examples/simple.sh` + +And the resulting image (svg): + +Simple Excalidraw Diagram as a SVG CLI usage help: diff --git a/README.md.jinja2 b/README.md.jinja2 index d5644a5..351b840 100644 --- a/README.md.jinja2 +++ b/README.md.jinja2 @@ -45,26 +45,28 @@ npm install # Might prompt for root. npx playwright install-deps -npx playwright install +npx playwright install firefox ``` Example: - +And the resulting image (svg): + +Simple Excalidraw Diagram as a SVG + CLI usage help: diff --git a/README.simple.sh.generated.svg b/README.simple.sh.generated.svg deleted file mode 100644 index 046911b..0000000 --- a/README.simple.sh.generated.svg +++ /dev/null @@ -1,369 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -$ . -/examples/ -simple.sh - - - - - - -npx excalidraw-brute-export-cli \ - - -  -i ./examples/simple.excalidraw \ - - -  --background 1 \ - - -  --embed-scene 1 \ - - -  --dark-mode 0 \ - - -  --scale 1 \ - - -  --format png \ - - -  -o "${PWD}/.deleteme/output.png" \ - - -  --verbose - - -+ npx excalidraw-brute-export-cli -i ./examples/simple.excalidraw --background 1 --embed-scene 1 --dark-mode 0 --scale 1 --format png -o  - - -/mnt/c/gdrive/code/excalidraw_brute_export/.deleteme/output.png --verbose - - -console.log('amain') - - -info -: logger.info('amain') - - -info -: options: - - -info -: { - - -  i:  -'./examples/simple.excalidraw' -, - - -  input:  -'./examples/simple.excalidraw' -, - - -  background:  -true -, - - -  b:  -true -, - - -  embedScene:  -true -, - - -  e:  -true -, - - -  darkMode:  -false -, - - -  d:  -false -, - - -  scale:  -1 -, - - -  s:  -1 -, - - -  format:  -'png' -, - - -  f:  -'png' -, - - -  o:  -'/mnt/c/gdrive/code/excalidraw_brute_export/.deleteme/output.png' -, - - -  output:  -'/mnt/c/gdrive/code/excalidraw_brute_export/.deleteme/output.png' -, - - -  verbose:  -true -, - - -  url:  -'https://excalidraw.com/' -, - - -  headless:  -true -, - - -  screenshots:  -'' - - -} - - -info -: OK file is chosen? - - -info -: fileChooserPromise! - - -info -: await exportScaleElements.count(): 3 - - -info -: fileChooserPromise! - - -info -: closing... - - -program.run() done - - - - -ls "${PWD}/.deleteme/output.png" - - -+ ls /mnt/c/gdrive/code/excalidraw_brute_export/.deleteme/output.png - - -/mnt/c/gdrive/code/excalidraw_brute_export/.deleteme/output.png - - - - - \ No newline at end of file diff --git a/examples/simple.sh b/examples/simple.sh index 96dea21..bb6eab0 100755 --- a/examples/simple.sh +++ b/examples/simple.sh @@ -10,7 +10,7 @@ npx excalidraw-brute-export-cli \ --dark-mode 0 \ --scale 1 \ --format png \ - -o "${PWD}/.deleteme/output.png" \ + -o "./.deleteme/output.png" \ --verbose -ls "${PWD}/.deleteme/output.png" +ls "./.deleteme/output.png" diff --git a/package-lock.json b/package-lock.json index 1540ecd..67beb30 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "excalidraw-brute-export-cli", - "version": "0.1.1", + "version": "0.2.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "excalidraw-brute-export-cli", - "version": "0.1.1", + "version": "0.2.0", "dependencies": { "@caporal/core": "^2.0.7", "playwright": "^1.43.0" @@ -424,19 +424,6 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, - "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, "node_modules/genversion": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/genversion/-/genversion-3.2.0.tgz", @@ -815,11 +802,11 @@ } }, "node_modules/playwright": { - "version": "1.43.0", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.43.0.tgz", - "integrity": "sha512-SiOKHbVjTSf6wHuGCbqrEyzlm6qvXcv7mENP+OZon1I07brfZLGdfWV0l/efAzVx7TF3Z45ov1gPEkku9q25YQ==", + "version": "1.43.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.43.1.tgz", + "integrity": "sha512-V7SoH0ai2kNt1Md9E3Gwas5B9m8KR2GVvwZnAI6Pg0m3sh7UvgiYhRrhsziCmqMJNouPckiOhk8T+9bSAK0VIA==", "dependencies": { - "playwright-core": "1.43.0" + "playwright-core": "1.43.1" }, "bin": { "playwright": "cli.js" @@ -832,9 +819,9 @@ } }, "node_modules/playwright-core": { - "version": "1.43.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.43.0.tgz", - "integrity": "sha512-iWFjyBUH97+pUFiyTqSLd8cDMMOS0r2ZYz2qEsPjH8/bX++sbIJT35MSwKnp1r/OQBAqC5XO99xFbJ9XClhf4w==", + "version": "1.43.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.43.1.tgz", + "integrity": "sha512-EI36Mto2Vrx6VF7rm708qSnesVQKbxEWvPrfA1IPY6HgczBplDx7ENtx+K2n4kJ41sLLkuGfmb0ZLSSXlDhqPg==", "bin": { "playwright-core": "cli.js" }, @@ -1569,12 +1556,6 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, - "fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "optional": true - }, "genversion": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/genversion/-/genversion-3.2.0.tgz", @@ -1869,18 +1850,18 @@ "dev": true }, "playwright": { - "version": "1.43.0", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.43.0.tgz", - "integrity": "sha512-SiOKHbVjTSf6wHuGCbqrEyzlm6qvXcv7mENP+OZon1I07brfZLGdfWV0l/efAzVx7TF3Z45ov1gPEkku9q25YQ==", + "version": "1.43.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.43.1.tgz", + "integrity": "sha512-V7SoH0ai2kNt1Md9E3Gwas5B9m8KR2GVvwZnAI6Pg0m3sh7UvgiYhRrhsziCmqMJNouPckiOhk8T+9bSAK0VIA==", "requires": { "fsevents": "2.3.2", - "playwright-core": "1.43.0" + "playwright-core": "1.43.1" } }, "playwright-core": { - "version": "1.43.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.43.0.tgz", - "integrity": "sha512-iWFjyBUH97+pUFiyTqSLd8cDMMOS0r2ZYz2qEsPjH8/bX++sbIJT35MSwKnp1r/OQBAqC5XO99xFbJ9XClhf4w==" + "version": "1.43.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.43.1.tgz", + "integrity": "sha512-EI36Mto2Vrx6VF7rm708qSnesVQKbxEWvPrfA1IPY6HgczBplDx7ENtx+K2n4kJ41sLLkuGfmb0ZLSSXlDhqPg==" }, "prettier": { "version": "3.2.5", diff --git a/scripts/generate-readme.sh b/scripts/generate-readme.sh index 63145e2..729798e 100755 --- a/scripts/generate-readme.sh +++ b/scripts/generate-readme.sh @@ -1,5 +1,4 @@ #!/bin/bash - # https://gist.github.com/mohanpedala/1e2ff5661761d3abd0385e8223e16425 set -e -x -v -u -o pipefail @@ -19,6 +18,29 @@ PYTHON_VERSION_PATH=${PWD}/scripts/.python-version \ bash scripts/format.sh +# FORCE_COLOR and TERM are set, to produce consistent results across different +# systems. +# +# Explanation: +# +# @caporal/core is used for argument parsing. +# +# chalk/chalk, via @caporal/core, uses an internal copy of the supports-color +# library (https://github.com/chalk/supports-color) here: +# . +# +# supports-color is terminal-emulator-dependent. +# +# But we want it to produce consistent results for the README output. +# +# To do this, we set the FORCE_COLOR environment variable to 3. +# +# However, this older version of supports-color does not always listen to +# FORCE_COLOR, so we also have to set TERM to 'dumb'. +export FORCE_COLOR=3 +export TERM=dumb + + rm -f "${PROJ_PATH}/README.md" || true touch "${PROJ_PATH}/README.md" python -m snipinator.cli \ diff --git a/scripts/pre.sh b/scripts/pre.sh index 1c478d1..8317d7b 100755 --- a/scripts/pre.sh +++ b/scripts/pre.sh @@ -5,14 +5,52 @@ set -e -x -v -u -o pipefail SCRIPT_DIR=$(realpath "$(dirname "${BASH_SOURCE[0]}")") source "${SCRIPT_DIR}/utilities/common.sh" +export PYTHON_VERSION_PATH="${PWD}/scripts/.python-version" +export TOML="${PWD}/scripts/pyproject.toml" +# This variable will be 1 when we are the ideal version in the GH action matrix. +IDEAL="0" +WANTED_NODE_VERSION=$(cat .nvmrc) +if [[ "${WANTED_NODE_VERSION}" == "v20.12.1" ]]; then + IDEAL="1" +fi + +if [[ -z "${GITHUB_ACTIONS:-}" ]]; then + if [[ "${IDEAL}" != "1" ]]; then + echo -e "${RED}Somehow we are not 'ideal' outside of GH Action workflow. IDEAL is meant to be 1 when we are running the GH Action matrix configuration that matches the configuration outside of the GH Action workflow. But we are not in the GH Action workflow, yet it is not 1!${NC}" + exit 1 + fi +fi + +ACTUAL_NODE_VERSION=$(node --version) +if [[ "${ACTUAL_NODE_VERSION}" != "${WANTED_NODE_VERSION}" ]]; then + echo -e "${RED}Node version is not as expected. Wanted: ${WANTED_NODE_VERSION}, Actual: ${ACTUAL_NODE_VERSION}${NC}" + exit 1 +fi + +# Check that no changes occurred to files through the workflow. +if [[ "${IDEAL}" == "1" ]]; then + STEP=pre bash scripts/utilities/changeguard.sh +fi + +npm install + + +EXTRA=dev bash scripts/utilities/pin-extra-reqs.sh npm run genversion bash scripts/run-all-examples.sh bash scripts/format.sh -bash scripts/generate-readme.sh bash scripts/run-ood-smoke-test.sh +bash scripts/generate-readme.sh if [[ -z "${GITHUB_ACTIONS:-}" ]]; then bash scripts/act.sh bash scripts/precommit.sh fi + +# Check that no changes occurred to files throughout pre.sh to tracked files. If +# changes occurred, they should be staged and pre.sh should be run again. +if [[ "${IDEAL}" == "1" ]]; then + STEP=post bash scripts/utilities/changeguard.sh +fi + echo -e "${GREEN}Success: pre.sh${NC}" diff --git a/scripts/pyproject.toml b/scripts/pyproject.toml index e10ac01..73452c4 100644 --- a/scripts/pyproject.toml +++ b/scripts/pyproject.toml @@ -1,4 +1,3 @@ - [project] name = "excalidraw_brute_export_cli_dev" version = "0.1.0" @@ -14,6 +13,32 @@ prod = [] # new dependency here, add the unpinned package name here, and then run # `EXTRA=dev bash scripts/utilties/pin-extra-reqs.sh`. dev = [ - "snipinator==1.2.0", + "cfgv==3.4.0", + "changeguard==0.3.1", + "colorama==0.4.6", + "defusedxml==0.7.1", + "distlib==0.3.8", + "filelock==3.13.4", + "identify==2.5.35", + "jinja2==3.1.3", + "markdown-it-py==3.0.0", + "markupsafe==2.1.5", + "mdurl==0.1.2", + "nodeenv==1.8.0", + "pathspec==0.12.1", + "pexpect==4.9.0", + "platformdirs==4.2.0", "pre-commit==3.5.0", + "ptyprocess==0.7.0", + "pygments==2.17.2", + "pyyaml==6.0.1", + "rich==13.7.1", + "rich-argparse==1.4.0", + "snipinator==1.2.0", + "toml-sort==0.23.1", + "tomlkit==0.12.4", + "types-colorama==0.4.15.20240311", + "types-pyyaml==6.0.12.20240311", + "typing-extensions==4.11.0", + "virtualenv==20.25.3" ] diff --git a/scripts/run-all-examples.sh b/scripts/run-all-examples.sh index 745b164..1b5c78f 100755 --- a/scripts/run-all-examples.sh +++ b/scripts/run-all-examples.sh @@ -5,6 +5,14 @@ set -e -x -v -u -o pipefail SCRIPT_DIR=$(realpath "$(dirname "${BASH_SOURCE[0]}")") source "${SCRIPT_DIR}/utilities/common.sh" + +# FORCE_COLOR and TERM are set, to produce consistent results across different +# systems. +# +# See generate-readme.sh for explanation. +export FORCE_COLOR=3 +export TERM=dumb + # For each sh in examples find examples -type f -name "*.sh" -print0 | while IFS= read -r -d '' EXAMPLE; do bash "${EXAMPLE}" diff --git a/scripts/run-ood-smoke-test.sh b/scripts/run-ood-smoke-test.sh index 99af090..fdfc3a0 100755 --- a/scripts/run-ood-smoke-test.sh +++ b/scripts/run-ood-smoke-test.sh @@ -42,6 +42,8 @@ cd "${TMP_DIR}" npm install "${TARBALL}" echo -e "${GREEN}Success: excalidraw-brute-export-cli installed successfully${NC}" +npx playwright install firefox + npx --no-install excalidraw-brute-export-cli --version npx --no-install excalidraw-brute-export-cli --help echo -e "${GREEN}Success: excalidraw-brute-export-cli smoke test ran successfully${NC}" diff --git a/scripts/utilities/changeguard.sh b/scripts/utilities/changeguard.sh new file mode 100755 index 0000000..dcfdfb9 --- /dev/null +++ b/scripts/utilities/changeguard.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# This script is used to check that no changes occur to files through the act +# workflow. If changes occurred, they should have been staged. This is usually +# run after a `git checkout-index --all` which extracts all the staged files, +# and before the main body of `pre.sh`. + +# https://gist.github.com/mohanpedala/1e2ff5661761d3abd0385e8223e16425 +set -e -x -v -u -o pipefail + +SCRIPT_DIR=$(realpath "$(dirname "${BASH_SOURCE[0]}")") +source "${SCRIPT_DIR}/common.sh" + +TOML=${TOML:-"${PROJ_PATH}/pyproject.toml"} + +VENV_PATH="${PWD}/.cache/scripts/.venv" source "${PROJ_PATH}/scripts/utilities/ensure-venv.sh" +TOML=${TOML} EXTRA=dev \ + DEV_VENV_PATH="${PWD}/.cache/scripts/.venv" \ + TARGET_VENV_PATH="${PWD}/.cache/scripts/.venv" \ + bash "${PROJ_PATH}/scripts/utilities/ensure-reqs.sh" + +STEP=${STEP:-} + +METHOD=${METHOD:-auto} +AUDIT_FILE=${AUDIT_FILE:-"${PWD}/.cache/scripts/check-changes-audit.yaml"} +AUDIT_LOG=${AUDIT_LOG:-"${PWD}/.cache/scripts/check-changes-audit.log"} + +function do_hash() { + python -m changeguard.cli hash \ + --ignorefile "${PROJ_PATH}/.gitignore" \ + --ignoreline .trunk --ignoreline .git \ + --method "${METHOD}" \ + --tmp-backup-dir "${PWD}/.cache/scripts/audit-original" \ + --audit-file "${AUDIT_FILE}" \ + --directory "${PROJ_PATH}" +} + +function do_audit() { + python -m changeguard.cli audit \ + --audit-file "${AUDIT_FILE}" \ + --show-delta \ + --directory "${PROJ_PATH}" 2>&1 | tee "${AUDIT_LOG}" +} + +if [[ "${STEP}" == "pre" ]]; then + do_hash + echo -e "${GREEN}Hashes generated, saved to ${AUDIT_FILE}${NC}" +elif [[ "${STEP}" == "post" ]]; then + # trunk-ignore(shellcheck/SC2310) + if do_audit; then + echo -e "${GREEN}No changes occurred during the workflow${NC}" + else + echo -e "${RED}Error auditing files.${NC}" + echo -e "${RED}If a post-stage change occurred, you can ignore it by adding it to ${PROJ_PATH}/.gitignore or ${PROJ_PATH}/.changeguard-gitignore.${NC}" + exit 1 + fi +else + echo -e "${RED}Error: STEP must be set to 'pre' or 'post'${NC}" + exit 1 +fi diff --git a/scripts/utilities/pin-extra-reqs.py b/scripts/utilities/pin-extra-reqs.py new file mode 100644 index 0000000..2546ebd --- /dev/null +++ b/scripts/utilities/pin-extra-reqs.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +import argparse +from pathlib import Path +from typing import List, Tuple + +# tomlkit is used, so that everything is preserved, e.g comments etc. +import tomlkit +import tomlkit.container +import tomlkit.items +from tomlkit.toml_document import TOMLDocument + +_VALID_EXTRA_NAMES = ['dev', 'prod'] + +_DESCRIPTION = f""" +Pin the {{{",".join(_VALID_EXTRA_NAMES)}}} requirements in pyproject.toml. + +The purpose of this program is to take the output of +`pip-compile --extra EXTRA pyproject.toml` and reinsert the exactly +to-be-installed version numbers back into the updated pyproject.toml. + +This ensures that there is a valid, and consistent version of dependencies that +are known to work. +""" +parser = argparse.ArgumentParser(description=_DESCRIPTION) +parser.add_argument('--toml', + type=Path, + required=True, + help='Path to the pyproject.toml file') +parser.add_argument('--extra', + choices=_VALID_EXTRA_NAMES, + required=True, + help='Which extra requirements to pin (dev/prod)') +parser.add_argument( + '--reqs', + type=Path, + required=True, + help='Path to the generated requirements.txt file (from pip-compile)') +args = parser.parse_args() + +pyproject_path: Path = args.toml +requirements_path: Path = args.reqs +extra_name: str = args.extra + +pyproject_data: TOMLDocument = tomlkit.loads(pyproject_path.read_text()) +lines = requirements_path.read_text().splitlines() + + +def _StripContinuation(line) -> Tuple[bool, str]: + line = line.strip() + if line.endswith('\\'): + return (True, line[:-1].strip()) + return (False, line) + + +existing_dependencies: List[str] = [] +is_continuation = False +for i in range(len(lines)): + line = lines[i] + append_to_last = is_continuation + is_continuation, stripped_line = _StripContinuation(line) + if not stripped_line: + continue + if stripped_line.startswith('#'): + continue + if stripped_line.startswith('--'): + continue + if append_to_last: + existing_dependencies[-1] += stripped_line + else: + existing_dependencies.append(stripped_line) + +if 'project' not in pyproject_data: + raise ValueError('Invalid pyproject.toml file, missing "project" section.') +project_item = pyproject_data['project'] +if not isinstance(project_item, tomlkit.items.Table): + raise ValueError( + f'Invalid pyproject.toml file, expected "project" to be a table. Got {type(project_item)}.' + ) + +if 'optional-dependencies' not in project_item: + raise ValueError( + 'Invalid pyproject.toml file, expected "project" to have an entry for "optional-dependencies".' + ) +opt_deps = project_item['optional-dependencies'] +if not isinstance(opt_deps, tomlkit.items.Table): + raise ValueError( + f'Invalid pyproject.toml file, expected "project.optional-dependencies" to be a table. Got {type(opt_deps)}.' + ) + +if extra_name not in opt_deps: + raise ValueError( + f'Invalid pyproject.toml file, expected "project.optional-dependencies" to contain {extra_name}.' + ) +toml_extra_dependencies = opt_deps[extra_name] +if not isinstance(toml_extra_dependencies, tomlkit.items.Array): + raise ValueError( + f'Invalid pyproject.toml file, expected "project.optional-dependencies.{extra_name}" to be an array. Got {type(toml_extra_dependencies)}.' + ) +if sorted(list(toml_extra_dependencies)) == sorted(existing_dependencies): + print('No changes detected') + exit(0) + +toml_extra_dependencies.clear() +for dep in existing_dependencies: + toml_extra_dependencies.append(dep) +opt_deps[extra_name] = toml_extra_dependencies.multiline(True) + +output = tomlkit.dumps(pyproject_data) +if output == pyproject_path.read_text(): + print('No changes detected') + exit(0) + +# Write the updated pyproject.toml back to disk +pyproject_path.write_text(tomlkit.dumps(pyproject_data)) diff --git a/scripts/utilities/pin-extra-reqs.sh b/scripts/utilities/pin-extra-reqs.sh new file mode 100755 index 0000000..40e236b --- /dev/null +++ b/scripts/utilities/pin-extra-reqs.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# https://gist.github.com/mohanpedala/1e2ff5661761d3abd0385e8223e16425 +set -e -x -v -u -o pipefail + +SCRIPT_DIR=$(realpath "$(dirname "${BASH_SOURCE[0]}")") +source "${SCRIPT_DIR}/common.sh" + +TOML="${TOML:-}" +if [[ -z "${TOML}" ]]; then + echo -e "${RED}TOML must be set${NC}" + [[ $(realpath "$0"||true) == $(realpath "${BASH_SOURCE[0]}"||true) ]] && EXIT="exit" || EXIT="return" + ${EXIT} 1 +fi + +VENV_PATH="${PWD}/.cache/scripts/.venv" source "${PROJ_PATH}/scripts/utilities/ensure-venv.sh" +TOML=${TOML} EXTRA=dev \ + DEV_VENV_PATH="${PWD}/.cache/scripts/.venv" \ + TARGET_VENV_PATH="${PWD}/.cache/scripts/.venv" \ + bash "${PROJ_PATH}/scripts/utilities/ensure-reqs.sh" + + + +EXTRA=${EXTRA:-} + +if [[ "${EXTRA}" == "dev" ]]; then + : +elif [[ "${EXTRA}" == "prod" ]]; then + : +else + echo -e "${RED}EXTRA must be set to 'dev' or 'prod'${NC}" + [[ $(realpath "$0"||true) == $(realpath "${BASH_SOURCE[0]}"||true) ]] && EXIT="exit" || EXIT="return" + ${EXIT} 1 +fi + +################################################################################ +PINNED_REQ_FILE="${PWD}/.cache/scripts/${EXTRA}-requirements.pinned.txt" +PINNED_REQ_TOUCH_FILE="${PWD}/.cache/scripts/${EXTRA}-requirements.pinned.touch" +TOML_UPDATE_TOUCH_FILE="${PWD}/.cache/scripts/${EXTRA}-pyproject.toml.pinned.touch" +################################################################################ +# Extract the requirements from the pyproject.toml file into a requirements +# file. +# +# Check if we already did this. +export FILE="${TOML}" +export TOUCH_FILE="${PINNED_REQ_TOUCH_FILE}" +if bash "${PROJ_PATH}/scripts/utilities/is_not_dirty.sh"; then + echo -e "${GREEN}Requirements ${PINNED_REQ_FILE} are up to date${NC}" + [[ $(realpath "$0"||true) == $(realpath "${BASH_SOURCE[0]}"||true) ]] && EXIT="exit" || EXIT="return" +else + echo -e "${BLUE}Requirements ${PINNED_REQ_FILE} need updating${NC}" + echo -e "${BLUE}Generating ${PINNED_REQ_FILE}${NC}" + + PINNED_REQ_DIR=$(dirname "${PINNED_REQ_FILE}") + mkdir -p "${PINNED_REQ_DIR}" + python -m piptools compile --generate-hashes \ + --extra "${EXTRA}" \ + "${TOML}" \ + -o "${PINNED_REQ_FILE}" + echo -e "${GREEN}Generated ${PINNED_REQ_FILE}${NC}" + + export FILE="${TOML}" + export TOUCH_FILE="${PINNED_REQ_TOUCH_FILE}" + bash "${PROJ_PATH}/scripts/utilities/mark_dirty.sh" +fi +################################################################################ +# Pin extra requirements in pyproject.toml file. + +export FILE="${TOML}" +export TOUCH_FILE="${TOML_UPDATE_TOUCH_FILE}" +if bash "${PROJ_PATH}/scripts/utilities/is_not_dirty.sh"; then + echo -e "${GREEN}pyproject.toml is up to date${NC}" +else + echo -e "${BLUE}Altering pyproject.toml${NC}" + python "${PROJ_PATH}/scripts/utilities/pin-extra-reqs.py" \ + --reqs "${PINNED_REQ_FILE}" \ + --extra "${EXTRA}" \ + --toml "${TOML}" + ################################################################################ + # Format the pyproject.toml file + if toml-sort "${TOML}" --check; then + echo -e "${GREEN}pyproject.toml needs no formatting${NC}" + else + echo -e "${BLUE}pyproject.toml needs formatting${NC}" + toml-sort --in-place "${TOML}" + echo -e "${GREEN}pyproject.toml formatted${NC}" + fi + if toml-sort "${TOML}" --check; then + echo -e "${GREEN}pyproject.toml is formatted${NC}" + else + echo -e "${RED}pyproject.toml is not formatted${NC}" + [[ $(realpath "$0"||true) == $(realpath "${BASH_SOURCE[0]}"||true) ]] && EXIT="exit" || EXIT="return" + ${EXIT} 1 + fi + ################################################################################ + export FILE="${TOML}" + export TOUCH_FILE="${TOML_UPDATE_TOUCH_FILE}" + bash "${PROJ_PATH}/scripts/utilities/mark_dirty.sh" + echo -e "${GREEN}Pinned ${EXTRA} requirements${NC}" +fi