From 6ce4e2cac0efc1236453f20ab2381d446efbffe2 Mon Sep 17 00:00:00 2001 From: Peter Van Dyken Date: Wed, 13 Dec 2023 14:05:26 -0500 Subject: [PATCH] Allow builtins and resolved paths as cli types The architecture for type conversion in the argparse integration was intended for using pathlib.Path, but had no flexibility even for builtin types like int Here, remove the old `globals()` lookup, which exposed too much internal api. Instead, attempt to lookup simple strings as builtins, and strings with dots as resolved module imports (e.g. `module.submodule.type`). `Path` is kept as an exception, so using `type: Path` will still convert the argument into a path ruamel is now used for serialization, as pyyaml didn't support serialization of specific python classes (e.g. Path) without restricting other classes: it was all or nothing. ruamel is safe by default, and allows specifically opting in to representing different classes Clarify in docs how Path should be used for all paths to avoid relative path issues Resolves #294 --- docs/bids_app/config.md | 23 +++++- poetry.lock | 115 ++++++++++++++++++++++----- pyproject.toml | 2 +- snakebids/app.py | 4 +- snakebids/cli.py | 31 ++++++-- snakebids/io/config.py | 43 ++++++++++ snakebids/io/yaml.py | 19 +++++ snakebids/paths/specs.py | 6 +- snakebids/tests/test_app.py | 2 +- snakebids/tests/test_cli.py | 50 +++++++++++- snakebids/tests/test_yaml.py | 73 +++++++++++++++++ snakebids/utils/output.py | 63 +-------------- snakebids/utils/utils.py | 2 +- typings/pyfakefs/fake_filesystem.pyi | 8 +- typings/pyfakefs/helpers.pyi | 18 +++-- 15 files changed, 350 insertions(+), 109 deletions(-) create mode 100644 snakebids/io/config.py create mode 100644 snakebids/io/yaml.py create mode 100644 snakebids/tests/test_yaml.py diff --git a/docs/bids_app/config.md b/docs/bids_app/config.md index d095ef36..fe25c8fa 100644 --- a/docs/bids_app/config.md +++ b/docs/bids_app/config.md @@ -79,7 +79,28 @@ A mapping from the name of each ``analysis_level`` to the list of rules or files ### `parse_args` -A dictionary of command-line parameters to make available as part of the BIDS app. Each item of the mapping is passed to [argparse's add_argument function](#argparse.ArgumentParser.add_argument). A number of default entries are present in a new snakebids project's config file that structure the BIDS app's CLI, but additional command-line arguments can be added as necessary. +A dictionary of command-line parameters to make available as part of the BIDS app. Each item of the mapping is passed to [argparse's `add_argument` function](#argparse.ArgumentParser.add_argument). A number of default entries are present in a new snakebids project's config file that structure the BIDS app's CLI, but additional command-line arguments can be added as necessary. + +As in [`ArgumentParser.add_argument()`](#argparse.ArgumentParser.add_argument), `type` may be used to convert the argument to the specified type. It may be set to any type that can be serialized into yaml, for instance, `str`, `int`, `float`, and `boolean`. + +```yaml +parse_args: + --a-string: + help: args are string by default + --a-path: + help: | + A path pointing to data needed for the pipeline. These are still converted + into strings, but are first resolved into absolute paths (see below) + type: Path + --another-path: + help: This type annotation does the same thing as above + type: pathlib.Path + --a-number: + help: A number important for the analysis + type: float +``` + +When CLI parameters are used to collect paths, `type` should be set to [`Path`](#pathlib.Path) (or [`pathlib.Path`](#pathlib.Path)). These arguments will still be serialized as strings (since yaml doesn't have a path type), but snakebids will automatically resolve all arguments into absolute paths. This is important to prevent issues with snakebids and relative paths. ### `debug` diff --git a/poetry.lock b/poetry.lock index fd2fe130..ef591713 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2550,30 +2550,107 @@ files = [ {file = "rpds_py-0.16.2.tar.gz", hash = "sha256:781ef8bfc091b19960fc0142a23aedadafa826bc32b433fdfe6fd7f964d7ef44"}, ] +[[package]] +name = "ruamel-yaml" +version = "0.18.5" +description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruamel.yaml-0.18.5-py3-none-any.whl", hash = "sha256:a013ac02f99a69cdd6277d9664689eb1acba07069f912823177c5eced21a6ada"}, + {file = "ruamel.yaml-0.18.5.tar.gz", hash = "sha256:61917e3a35a569c1133a8f772e1226961bf5a1198bea7e23f06a0841dea1ab0e"}, +] + +[package.dependencies] +"ruamel.yaml.clib" = {version = ">=0.2.7", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.13\""} + +[package.extras] +docs = ["mercurial (>5.7)", "ryd"] +jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] + +[[package]] +name = "ruamel-yaml-clib" +version = "0.2.8" +description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" +optional = false +python-versions = ">=3.6" +files = [ + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:07238db9cbdf8fc1e9de2489a4f68474e70dffcb32232db7c08fa61ca0c7c462"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fff3573c2db359f091e1589c3d7c5fc2f86f5bdb6f24252c2d8e539d4e45f412"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:aa2267c6a303eb483de8d02db2871afb5c5fc15618d894300b88958f729ad74f"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:840f0c7f194986a63d2c2465ca63af8ccbbc90ab1c6001b1978f05119b5e7334"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:024cfe1fc7c7f4e1aff4a81e718109e13409767e4f871443cbff3dba3578203d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win32.whl", hash = "sha256:c69212f63169ec1cfc9bb44723bf2917cbbd8f6191a00ef3410f5a7fe300722d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win_amd64.whl", hash = "sha256:cabddb8d8ead485e255fe80429f833172b4cadf99274db39abc080e068cbcc31"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bef08cd86169d9eafb3ccb0a39edb11d8e25f3dae2b28f5c52fd997521133069"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:b16420e621d26fdfa949a8b4b47ade8810c56002f5389970db4ddda51dbff248"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:25c515e350e5b739842fc3228d662413ef28f295791af5e5110b543cf0b57d9b"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_24_aarch64.whl", hash = "sha256:1707814f0d9791df063f8c19bb51b0d1278b8e9a2353abbb676c2f685dee6afe"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:46d378daaac94f454b3a0e3d8d78cafd78a026b1d71443f4966c696b48a6d899"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:09b055c05697b38ecacb7ac50bdab2240bfca1a0c4872b0fd309bb07dc9aa3a9"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win32.whl", hash = "sha256:53a300ed9cea38cf5a2a9b069058137c2ca1ce658a874b79baceb8f892f915a7"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win_amd64.whl", hash = "sha256:c2a72e9109ea74e511e29032f3b670835f8a59bbdc9ce692c5b4ed91ccf1eedb"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ebc06178e8821efc9692ea7544aa5644217358490145629914d8020042c24aa1"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:edaef1c1200c4b4cb914583150dcaa3bc30e592e907c01117c08b13a07255ec2"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d176b57452ab5b7028ac47e7b3cf644bcfdc8cacfecf7e71759f7f51a59e5c92"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_24_aarch64.whl", hash = "sha256:1dc67314e7e1086c9fdf2680b7b6c2be1c0d8e3a8279f2e993ca2a7545fecf62"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3213ece08ea033eb159ac52ae052a4899b56ecc124bb80020d9bbceeb50258e9"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aab7fd643f71d7946f2ee58cc88c9b7bfc97debd71dcc93e03e2d174628e7e2d"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win32.whl", hash = "sha256:5c365d91c88390c8d0a8545df0b5857172824b1c604e867161e6b3d59a827eaa"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win_amd64.whl", hash = "sha256:1758ce7d8e1a29d23de54a16ae867abd370f01b5a69e1a3ba75223eaa3ca1a1b"}, + {file = "ruamel.yaml.clib-0.2.8-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a5aa27bad2bb83670b71683aae140a1f52b0857a2deff56ad3f6c13a017a26ed"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c58ecd827313af6864893e7af0a3bb85fd529f862b6adbefe14643947cfe2942"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f481f16baec5290e45aebdc2a5168ebc6d35189ae6fea7a58787613a25f6e875"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_24_aarch64.whl", hash = "sha256:77159f5d5b5c14f7c34073862a6b7d34944075d9f93e681638f6d753606c6ce6"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7f67a1ee819dc4562d444bbafb135832b0b909f81cc90f7aa00260968c9ca1b3"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4ecbf9c3e19f9562c7fdd462e8d18dd902a47ca046a2e64dba80699f0b6c09b7"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:87ea5ff66d8064301a154b3933ae406b0863402a799b16e4a1d24d9fbbcbe0d3"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win32.whl", hash = "sha256:75e1ed13e1f9de23c5607fe6bd1aeaae21e523b32d83bb33918245361e9cc51b"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:3f215c5daf6a9d7bbed4a0a4f760f3113b10e82ff4c5c44bec20a68c8014f675"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1b617618914cb00bf5c34d4357c37aa15183fa229b24767259657746c9077615"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:a6a9ffd280b71ad062eae53ac1659ad86a17f59a0fdc7699fd9be40525153337"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:305889baa4043a09e5b76f8e2a51d4ffba44259f6b4c72dec8ca56207d9c6fe1"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:700e4ebb569e59e16a976857c8798aee258dceac7c7d6b50cab63e080058df91"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e2b4c44b60eadec492926a7270abb100ef9f72798e18743939bdbf037aab8c28"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e79e5db08739731b0ce4850bed599235d601701d5694c36570a99a0c5ca41a9d"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win32.whl", hash = "sha256:955eae71ac26c1ab35924203fda6220f84dce57d6d7884f189743e2abe3a9fbe"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win_amd64.whl", hash = "sha256:56f4252222c067b4ce51ae12cbac231bce32aee1d33fbfc9d17e5b8d6966c312"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:03d1162b6d1df1caa3a4bd27aa51ce17c9afc2046c31b0ad60a0a96ec22f8001"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba64af9fa9cebe325a62fa398760f5c7206b215201b0ec825005f1b18b9bccf"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:a1a45e0bb052edf6a1d3a93baef85319733a888363938e1fc9924cb00c8df24c"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:da09ad1c359a728e112d60116f626cc9f29730ff3e0e7db72b9a2dbc2e4beed5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:184565012b60405d93838167f425713180b949e9d8dd0bbc7b49f074407c5a8b"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a75879bacf2c987c003368cf14bed0ffe99e8e85acfa6c0bfffc21a090f16880"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win32.whl", hash = "sha256:84b554931e932c46f94ab306913ad7e11bba988104c5cff26d90d03f68258cd5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win_amd64.whl", hash = "sha256:25ac8c08322002b06fa1d49d1646181f0b2c72f5cbc15a85e80b4c30a544bb15"}, + {file = "ruamel.yaml.clib-0.2.8.tar.gz", hash = "sha256:beb2e0404003de9a4cab9753a8805a8fe9320ee6673136ed7f04255fe60bb512"}, +] + [[package]] name = "ruff" -version = "0.1.11" +version = "0.1.12" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.1.11-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:a7f772696b4cdc0a3b2e527fc3c7ccc41cdcb98f5c80fdd4f2b8c50eb1458196"}, - {file = "ruff-0.1.11-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:934832f6ed9b34a7d5feea58972635c2039c7a3b434fe5ba2ce015064cb6e955"}, - {file = "ruff-0.1.11-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea0d3e950e394c4b332bcdd112aa566010a9f9c95814844a7468325290aabfd9"}, - {file = "ruff-0.1.11-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9bd4025b9c5b429a48280785a2b71d479798a69f5c2919e7d274c5f4b32c3607"}, - {file = "ruff-0.1.11-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1ad00662305dcb1e987f5ec214d31f7d6a062cae3e74c1cbccef15afd96611d"}, - {file = "ruff-0.1.11-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:4b077ce83f47dd6bea1991af08b140e8b8339f0ba8cb9b7a484c30ebab18a23f"}, - {file = "ruff-0.1.11-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4a88efecec23c37b11076fe676e15c6cdb1271a38f2b415e381e87fe4517f18"}, - {file = "ruff-0.1.11-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5b25093dad3b055667730a9b491129c42d45e11cdb7043b702e97125bcec48a1"}, - {file = "ruff-0.1.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:231d8fb11b2cc7c0366a326a66dafc6ad449d7fcdbc268497ee47e1334f66f77"}, - {file = "ruff-0.1.11-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:09c415716884950080921dd6237767e52e227e397e2008e2bed410117679975b"}, - {file = "ruff-0.1.11-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:0f58948c6d212a6b8d41cd59e349751018797ce1727f961c2fa755ad6208ba45"}, - {file = "ruff-0.1.11-py3-none-musllinux_1_2_i686.whl", hash = "sha256:190a566c8f766c37074d99640cd9ca3da11d8deae2deae7c9505e68a4a30f740"}, - {file = "ruff-0.1.11-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:6464289bd67b2344d2a5d9158d5eb81025258f169e69a46b741b396ffb0cda95"}, - {file = "ruff-0.1.11-py3-none-win32.whl", hash = "sha256:9b8f397902f92bc2e70fb6bebfa2139008dc72ae5177e66c383fa5426cb0bf2c"}, - {file = "ruff-0.1.11-py3-none-win_amd64.whl", hash = "sha256:eb85ee287b11f901037a6683b2374bb0ec82928c5cbc984f575d0437979c521a"}, - {file = "ruff-0.1.11-py3-none-win_arm64.whl", hash = "sha256:97ce4d752f964ba559c7023a86e5f8e97f026d511e48013987623915431c7ea9"}, - {file = "ruff-0.1.11.tar.gz", hash = "sha256:f9d4d88cb6eeb4dfe20f9f0519bd2eaba8119bde87c3d5065c541dbae2b5a2cb"}, + {file = "ruff-0.1.12-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:544038693543c11edc56bb94a9875df2dc249e3616f90c15964c720dcccf0745"}, + {file = "ruff-0.1.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:8a0e3ef6299c4eab75a7740730e4b4bd4a36e0bd8102ded01553403cad088fd4"}, + {file = "ruff-0.1.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47f6d939461e3273f10f4cd059fd0b83c249d73f1736032fffbac83a62939395"}, + {file = "ruff-0.1.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25be18abc1fc3f3d3fb55855c41ed5d52063316defde202f413493bb3888218c"}, + {file = "ruff-0.1.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d41e9f100b50526d80b076fc9c103c729387ff3f10f63606ed1038c30a372a40"}, + {file = "ruff-0.1.12-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:472a0548738d4711549c7874b43fab61aacafb1fede29c5232d4cfb8e2d13f69"}, + {file = "ruff-0.1.12-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46685ef2f106b827705df876d38617741ed4f858bbdbc0817f94476c45ab6669"}, + {file = "ruff-0.1.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf6073749c70b616d7929897b14824ec6713a6c3a8195dfd2ffdcc66594d880c"}, + {file = "ruff-0.1.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4bdf26e5a2efab4c3aaf6b61648ea47a525dc12775810a85c285dc9ca03e5ac0"}, + {file = "ruff-0.1.12-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:b631c6a95e4b6d5c4299e599067b5a89f5b18e2f2d9a6c22b879b3c4b077c96e"}, + {file = "ruff-0.1.12-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f193f460e231e63af5fc7516897cf5ab257cbda72ae83cf9a654f1c80c3b758a"}, + {file = "ruff-0.1.12-py3-none-musllinux_1_2_i686.whl", hash = "sha256:718523c3a0b787590511f212d30cc9b194228ef369c8bdd72acd1282cc27c468"}, + {file = "ruff-0.1.12-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1c49e826de55d81a6ef93808b760925e492bad7cc470aaa114a3be158b2c7f99"}, + {file = "ruff-0.1.12-py3-none-win32.whl", hash = "sha256:fbb1c002eeacb60161e51d77b2274c968656599477a1c8c65066953276e8ee2b"}, + {file = "ruff-0.1.12-py3-none-win_amd64.whl", hash = "sha256:7fe06ba77e5b7b78db1d058478c47176810f69bb5be7c1b0d06876af59198203"}, + {file = "ruff-0.1.12-py3-none-win_arm64.whl", hash = "sha256:bb29f8e3e6c95024902eaec5a9ce1fd5ac4e77f4594f4554e67fbb0f6d9a2f37"}, + {file = "ruff-0.1.12.tar.gz", hash = "sha256:97189f38c655e573f6bea0d12e9f18aad5539fd08ab50651449450999f45383a"}, ] [[package]] @@ -3556,4 +3633,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8,<4.0" -content-hash = "8bbe64a495b36ab9a363700cdc361542ec810ad7a6dd6e8555dbfcf512dac729" +content-hash = "9110c94c7ca69d61ab44430e33cd66a2e4b030cf58fafc196105d2bb84b3d7cf" diff --git a/pyproject.toml b/pyproject.toml index bec94a84..ce90d129 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,6 @@ snakemake = [ { version = ">=5.28.0,<8", python = ">=3.8" }, { version = ">=7.18.2,<8", python = ">=3.11" }, ] -PyYAML = ">=6" typing-extensions = ">=3.10.0" attrs = ">=21.2.0" boutiques = "^0.5.25" @@ -70,6 +69,7 @@ copier = ">=8.2.0" jinja2-time = ">=0.2.0" # minimum 2.31.0 because of security vulnerability requests = ">=2.31.0" +ruamel-yaml = ">=0.17.2" [tool.poetry.group.dev.dependencies] pytest = "^7.0.0" diff --git a/snakebids/app.py b/snakebids/app.py index 81f18fd6..2d22d288 100644 --- a/snakebids/app.py +++ b/snakebids/app.py @@ -21,10 +21,10 @@ parse_snakebids_args, ) from snakebids.exceptions import ConfigError, RunError +from snakebids.io.config import write_config from snakebids.types import OptionalFilter from snakebids.utils.output import ( prepare_bidsapp_output, - write_config_file, write_output_mode, ) from snakebids.utils.utils import DEPRECATION_FLAG, to_resolved_path @@ -234,7 +234,7 @@ def run_snakemake(self) -> None: app = plugin(app) or app # Write the config file - write_config_file( + write_config( config_file=new_config_file, data=dict( app.config, diff --git a/snakebids/cli.py b/snakebids/cli.py index 75168190..9aef32a5 100644 --- a/snakebids/cli.py +++ b/snakebids/cli.py @@ -11,7 +11,8 @@ import snakemake from typing_extensions import override -from snakebids.exceptions import MisspecifiedCliFilterError +from snakebids.exceptions import ConfigError, MisspecifiedCliFilterError +from snakebids.io.yaml import get_yaml_io from snakebids.types import InputsConfig, OptionalFilter from snakebids.utils.utils import to_resolved_path @@ -204,6 +205,26 @@ def create_parser(include_snakemake: bool = False) -> argparse.ArgumentParser: return parser +def _find_type(name: str, *, yamlsafe: bool = True) -> type[Any]: + import importlib + + if name == "Path": + return Path + *module_name, obj_name = name.split(".") if "." in name else ("builtins", name) + try: + type_ = getattr(importlib.import_module(".".join(module_name)), obj_name) + except (ImportError, AttributeError) as err: + msg = f"{name} could not be resolved" + raise ConfigError(msg) from err + if not callable(type_): + msg = f"{name} cannot be used as a type" + raise ConfigError(msg) + if yamlsafe and type_ not in get_yaml_io().representer.yaml_representers: + msg = f"{name} cannot be serialized into yaml" + raise ConfigError(msg) + return type_ + + def add_dynamic_args( parser: argparse.ArgumentParser, parse_args: Mapping[str, Any], @@ -220,16 +241,12 @@ def add_dynamic_args( # a str to allow the edge case where it's already # been converted if "type" in arg: - try: - arg_dict = {**arg, "type": globals()[str(arg["type"])]} - except KeyError as err: - msg = f"{arg['type']} is not available as a type for {name}" - raise TypeError(msg) from err + arg_dict = {**arg, "type": _find_type(str(arg["type"]))} else: arg_dict = arg app_group.add_argument( *_make_underscore_dash_aliases(name), - **arg_dict, + **arg_dict, # type: ignore ) # general parser for diff --git a/snakebids/io/config.py b/snakebids/io/config.py new file mode 100644 index 00000000..9ac402ef --- /dev/null +++ b/snakebids/io/config.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import errno +import json +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from snakebids.io.yaml import get_yaml_io + +if TYPE_CHECKING: + from _typeshed import StrPath + + +def write_config( + config_file: StrPath, data: dict[str, Any], force_overwrite: bool = False +) -> None: + """Write provided data as yaml or json to provided path. + + Output type is decided based on file suffix: .json -> JSON, .yaml,.yml -> YAML + + Parameters + ---------- + config_file + Path of yaml file + data + Data to format + force_overwrite + If True, force overwrite of already existing files, otherwise error out + """ + config_file = Path(config_file) + if (config_file.exists()) and not force_overwrite: + err = FileExistsError( + errno.EEXIST, f"'{config_file}' already exists", str(config_file) + ) + raise err + config_file.parent.mkdir(exist_ok=True) + + if config_file.suffix == ".json": + with open(config_file, "w", encoding="utf-8") as f: + json.dump(data, f, indent=4) + return + + get_yaml_io().dump(data, config_file) diff --git a/snakebids/io/yaml.py b/snakebids/io/yaml.py new file mode 100644 index 00000000..8ecacfcb --- /dev/null +++ b/snakebids/io/yaml.py @@ -0,0 +1,19 @@ +from pathlib import Path, PosixPath, WindowsPath + +from ruamel.yaml import YAML, Dumper + + +def get_yaml_io(): + """Return yaml loader/dumper configured for snakebids.""" + yaml = YAML() + + # Represent any PathLikes as str. + def path2str(dumper: Dumper, data: Path): + return dumper.represent_scalar( + "tag:yaml.org,2002:str", + str(data), + ) + + yaml.representer.add_representer(PosixPath, path2str) + yaml.representer.add_representer(WindowsPath, path2str) + return yaml diff --git a/snakebids/paths/specs.py b/snakebids/paths/specs.py index 32a14cc1..a3ebd09f 100644 --- a/snakebids/paths/specs.py +++ b/snakebids/paths/specs.py @@ -1,8 +1,8 @@ import importlib_resources as impr import more_itertools as itx -import yaml from typing_extensions import NotRequired, TypeAlias, TypedDict +from snakebids.io.yaml import get_yaml_io from snakebids.paths import resources @@ -38,7 +38,9 @@ def v0_0_0(subject_dir: bool = True, session_dir: bool = True) -> BidsPathSpec: If False, downstream path generator will not include the session dir `*/ses-{session}/*` """ - spec = yaml.safe_load(impr.files(resources).joinpath("spec.0.0.0.yaml").read_text()) + spec = get_yaml_io().load( + impr.files(resources).joinpath("spec.0.0.0.yaml").read_text() + ) if not subject_dir: _find_entity(spec, "subject")["dir"] = False diff --git a/snakebids/tests/test_app.py b/snakebids/tests/test_app.py index 61698716..b451f092 100644 --- a/snakebids/tests/test_app.py +++ b/snakebids/tests/test_app.py @@ -121,7 +121,7 @@ def io_mocks(self, mocker: MockerFixture): return { "write_output_mode": mocker.patch.object(sn_app, "write_output_mode"), "prepare_output": mocker.patch.object(sn_app, "prepare_bidsapp_output"), - "write_config": mocker.patch.object(sn_app, "write_config_file"), + "write_config": mocker.patch.object(sn_app, "write_config"), "snakemake": mocker.patch.object(snakemake, "main"), } diff --git a/snakebids/tests/test_cli.py b/snakebids/tests/test_cli.py index c0180324..6edc446b 100644 --- a/snakebids/tests/test_cli.py +++ b/snakebids/tests/test_cli.py @@ -6,7 +6,6 @@ import sys from argparse import ArgumentParser, Namespace from collections.abc import Sequence -from os import PathLike from pathlib import Path from typing import ClassVar, Mapping @@ -21,6 +20,7 @@ create_parser, parse_snakebids_args, ) +from snakebids.exceptions import ConfigError from snakebids.tests import strategies as sb_st from snakebids.tests.helpers import allow_function_scoped from snakebids.types import InputsConfig, OptionalFilter @@ -198,7 +198,7 @@ def test_converts_type_path_into_pathlike( ): mocker.patch.object(sys, "argv", self.mock_all_args) args = parser.parse_args() - assert isinstance(args.derivatives[0], PathLike) + assert isinstance(args.derivatives[0], Path) def test_fails_if_undefined_type_given(self): parse_args_copy = copy.deepcopy(parse_args) @@ -206,9 +206,53 @@ def test_fails_if_undefined_type_given(self): "help": "Generic Help Message", "type": "UnheardClass", } - with pytest.raises(TypeError): + with pytest.raises(ConfigError, match="could not be resolved"): add_dynamic_args(create_parser(), parse_args_copy, pybids_inputs) + def test_convert_arg_to_builtin( + self, parser: ArgumentParser, mocker: MockerFixture + ): + new_args = { + "--new-param": { + "help": "Generic Help Message", + "type": "int", + } + } + mocker.patch.object(sys, "argv", [*self.mock_all_args, "--new-param", "12"]) + add_dynamic_args(parser, new_args, {}) + args = parser.parse_args() + assert isinstance(args.new_param, int) + + def test_non_serialiable_type_raises_error(self, parser: ArgumentParser): + new_args = { + "--new-param": { + "help": "Generic Help Message", + "type": "snakebids.utils.utils.BidsEntity", + } + } + with pytest.raises(ConfigError, match="cannot be serialized into yaml"): + add_dynamic_args(parser, new_args, {}) + + def test_using_module_as_type_gives_error(self, parser: ArgumentParser): + new_args = { + "--new-param": { + "help": "Generic Help Message", + "type": "snakebids.utils.utils", + } + } + with pytest.raises(ConfigError, match="cannot be used as a type"): + add_dynamic_args(parser, new_args, {}) + + def test_using_class_method_as_type_gives_error(self, parser: ArgumentParser): + new_args = { + "--new-param": { + "help": "Generic Help Message", + "type": "snakebids.utils.utils.BidsEntity.from_tag", + } + } + with pytest.raises(ConfigError, match="could not be resolved"): + add_dynamic_args(parser, new_args, {}) + def test_resolves_paths(self, parser: ArgumentParser, mocker: MockerFixture): mocker.patch.object(sys, "argv", self.mock_all_args) args = parse_snakebids_args(parser).args_dict diff --git a/snakebids/tests/test_yaml.py b/snakebids/tests/test_yaml.py new file mode 100644 index 00000000..da177398 --- /dev/null +++ b/snakebids/tests/test_yaml.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from io import StringIO +from pathlib import Path + +import pytest +from hypothesis import given +from hypothesis import strategies as st +from pyfakefs.fake_filesystem import FakeFilesystem +from pytest_mock import MockerFixture +from pytest_mock.plugin import MockType + +import snakebids.io.config as configio +import snakebids.io.yaml as yamlio +from snakebids.tests.helpers import allow_function_scoped + + +@given(path=st.text(min_size=1).map(Path)) +def test_paths_formatted_as_str(path: Path): + string = StringIO() + yaml = yamlio.get_yaml_io() + yaml.dump({"key": path}, string) + string.seek(0, 0) + assert yaml.load(string)["key"] == str(path) + + +class TestWriteConfig: + def io_mocks(self, mocker: MockerFixture) -> dict[str, MockType]: + return { + "mopen": mocker.patch.object(configio, "open", mocker.mock_open()), + "jsondump": mocker.patch.object(configio.json, "dump"), + "mkdir": mocker.patch.object(configio.Path, "mkdir"), + "yamldump": mocker.patch.object(yamlio.YAML, "dump"), + } + + @allow_function_scoped + @given( + ext=st.sampled_from([".json", ".yaml", ".yml"]), + path=st.text().map(Path).filter(lambda p: p != Path() and not p.exists()), + ) + def test_writes_correct_format(self, ext: str, path: Path, mocker: MockerFixture): + mocker.stopall() + mocks = self.io_mocks(mocker) + path = path.with_suffix(ext) + configio.write_config(path, {}) + if ext == ".json": + mocks["mopen"].assert_called_once_with(path, "w", encoding="utf-8") + mocks["jsondump"].assert_called_once() + mocks["yamldump"].assert_not_called() + else: + mocks["mopen"].assert_not_called() + mocks["jsondump"].assert_not_called() + mocks["yamldump"].assert_called_once_with({}, path) + + @allow_function_scoped + @given(path=st.text().filter(lambda s: s not in {".", ""})) + def test_doesnt_overwrite_file(self, path: str, fakefs: FakeFilesystem): + fakefs.reset() + fakefs.create_file(path) + with pytest.raises(FileExistsError, match="already exists"): + configio.write_config(path, {}) + + @allow_function_scoped + @given(path=st.text().filter(lambda s: s not in {".", ""})) + def test_overwrites_file_if_forced( + self, path: str, fakefs: FakeFilesystem, mocker: MockerFixture + ): + mocker.stopall() + fakefs.reset() + fakefs.create_file(path) + mocks = self.io_mocks(mocker) + configio.write_config(path, {}, force_overwrite=True) + assert mocks["jsondump"].call_count ^ mocks["yamldump"].call_count diff --git a/snakebids/utils/output.py b/snakebids/utils/output.py index d27dd6a9..70702819 100644 --- a/snakebids/utils/output.py +++ b/snakebids/utils/output.py @@ -3,12 +3,10 @@ from __future__ import annotations import json -from collections import OrderedDict -from pathlib import Path, PosixPath, WindowsPath -from typing import Any, Literal +from pathlib import Path +from typing import Literal import more_itertools as itx -import yaml from snakebids.exceptions import RunError @@ -134,60 +132,3 @@ def _get_snakebids_file(outputdir: Path) -> dict[str, str] | None: raise RunError( msg, ) - - -def write_config_file( - config_file: Path, data: dict[str, Any], force_overwrite: bool = False -) -> None: - """Write provided data as yaml to provided path. - - Parameters - ---------- - config_file - Path of yaml file - data - Data to format - force_overwrite - If True, force overwrite of already existing files, otherwise error out - """ - if (config_file.exists()) and not force_overwrite: - msg = ( - f"A config file named {config_file.name} already exists:\n" - f"\t- {config_file.resolve()}\n" - "Please move or rename either the existing or incoming config." - ) - raise RunError(msg) - config_file.parent.mkdir(exist_ok=True) - - # TODO: copy to a time-hashed file for provenance purposes? - # unused as of now.. - # time_hash = get_time_hash() - - with open(config_file, "w", encoding="utf-8") as f: - # write either as JSON or YAML - if config_file.suffix == ".json": - json.dump(data, f, indent=4) - return - - # if not json, then should be yaml or yml - - # this is needed to make the output yaml clean - yaml.add_representer( - OrderedDict, - lambda dumper, data: dumper.represent_mapping( # type: ignore - "tag:yaml.org,2002:map", - data.items(), # type: ignore - ), - ) - - # Represent any PathLikes as str. - def path2str(dumper, data): # type: ignore - return dumper.represent_scalar( # type: ignore - "tag:yaml.org,2002:str", - str(data), # type: ignore - ) - - yaml.add_representer(PosixPath, path2str) # type: ignore - yaml.add_representer(WindowsPath, path2str) # type: ignore - - yaml.dump(data, f, default_flow_style=False, sort_keys=False) diff --git a/snakebids/utils/utils.py b/snakebids/utils/utils.py index 62b98f93..199c7ff0 100644 --- a/snakebids/utils/utils.py +++ b/snakebids/utils/utils.py @@ -239,7 +239,7 @@ def __init__(self, path: str, entity: BidsEntity) -> None: class _Documented(Protocol): - __doc__: str # noqa: A003 + __doc__: str def property_alias( diff --git a/typings/pyfakefs/fake_filesystem.pyi b/typings/pyfakefs/fake_filesystem.pyi index 162b3a0e..037a020f 100644 --- a/typings/pyfakefs/fake_filesystem.pyi +++ b/typings/pyfakefs/fake_filesystem.pyi @@ -185,7 +185,7 @@ class FakeFilesystem: """Set the simulated type of operating system underlying the fake file system.""" ... - def reset(self, total_size: int | None = ...): # -> None: + def reset(self, total_size: int | None = ...) -> None: """Remove all file system contents and reset the root.""" ... def pause(self) -> None: @@ -352,7 +352,7 @@ class FakeFilesystem: times: tuple[int | float, int | float] | None = ..., *, ns: tuple[int, int] | None = ..., - follow_symlinks: bool = ... + follow_symlinks: bool = ..., ) -> None: """Change the access and modified times of a file. @@ -724,7 +724,7 @@ class FakeFilesystem: ... def create_file( self, - file_path: AnyPath, + file_path: AnyPath[AnyStr], st_mode: int = ..., contents: AnyString = ..., st_size: int | None = ..., @@ -732,7 +732,7 @@ class FakeFilesystem: apply_umask: bool = ..., encoding: str | None = ..., errors: str | None = ..., - side_effect: Callable | None = ..., + side_effect: Callable[[FakeFile], Any] | None = ..., ) -> FakeFile: """Create `file_path`, including all the parent directories along the way. diff --git a/typings/pyfakefs/helpers.pyi b/typings/pyfakefs/helpers.pyi index 8698da19..4f711301 100644 --- a/typings/pyfakefs/helpers.pyi +++ b/typings/pyfakefs/helpers.pyi @@ -4,13 +4,12 @@ This type stub file was generated by pyright. import io import os -import platform import sys from typing import Any, AnyStr, Optional, Union, overload """Helper classes use for fake file system implementation.""" AnyString = Union[str, bytes] -AnyPath = Union[AnyStr, "os.PathLike[str]", "os.PathLike[bytes]"] +AnyPath = Union[AnyStr, "os.PathLike[AnyStr]"] IS_PYPY = ... IS_WIN = ... IN_DOCKER = ... @@ -58,12 +57,14 @@ def is_int_type(val: Any) -> bool: def is_byte_string(val: Any) -> bool: """Return True if `val` is a bytes-like object, False for a unicode - string.""" + string. + """ ... def is_unicode_string(val: Any) -> bool: """Return True if `val` is a unicode string, False for a bytes-like - object.""" + object. + """ ... @overload @@ -73,12 +74,14 @@ def make_string_path(dir_name: os.PathLike) -> str: ... def make_string_path(dir_name: AnyPath) -> AnyStr: ... def to_string(path: Union[AnyStr, Union[str, bytes]]) -> str: """Return the string representation of a byte string using the preferred - encoding, or the string itself if path is a str.""" + encoding, or the string itself if path is a str. + """ ... def to_bytes(path: Union[AnyStr, Union[str, bytes]]) -> bytes: """Return the bytes representation of a string using the preferred - encoding, or the byte string itself if path is a byte string.""" + encoding, or the byte string itself if path is a byte string. + """ ... def join_strings(s1: AnyStr, s2: AnyStr) -> AnyStr: @@ -88,7 +91,8 @@ def join_strings(s1: AnyStr, s2: AnyStr) -> AnyStr: def real_encoding(encoding: Optional[str]) -> Optional[str]: """Since Python 3.10, the new function ``io.text_encoding`` returns "locale" as the encoding if None is defined. This will be handled - as no encoding in pyfakefs.""" + as no encoding in pyfakefs. + """ ... def now(): ...