diff --git a/conda/meta.yaml b/conda/meta.yaml index 08d35d09..75281ceb 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -14,6 +14,8 @@ requirements: - python>=3.10 - scipp>=24.02.0 - scippnexus>=24.03.0 + - scitacean + - paramiko test: imports: diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000..b8a27157 --- /dev/null +++ b/conftest.py @@ -0,0 +1,8 @@ +import pytest +from scitacean.testing.backend import add_pytest_option as add_backend_option + +pytest_plugins = ("scitacean.testing.backend.fixtures",) + + +def pytest_addoption(parser: pytest.Parser) -> None: + add_backend_option(parser) diff --git a/pyproject.toml b/pyproject.toml index 5e4e2b80..cfe16427 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,8 @@ requires-python = ">=3.10" dependencies = [ "scipp >= 24.02.0", "scippnexus >= 24.03.0", + "scitacean", + "paramiko", ] dynamic = ["version"] diff --git a/requirements/base.in b/requirements/base.in index d058df89..50c85aa1 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -4,3 +4,5 @@ # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! scipp >= 24.02.0 scippnexus >= 24.03.0 +scitacean +paramiko diff --git a/requirements/base.txt b/requirements/base.txt index 5e1a1dd3..5fafc3cf 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,26 +1,70 @@ -# SHA1:b5fdb6600edc83ab95fb0e848607edef52cdd293 +# SHA1:e88d1da9daae168a676da3510d3baae296449627 # # This file is autogenerated by pip-compile-multi # To update, run: # # pip-compile-multi # +annotated-types==0.6.0 + # via pydantic +bcrypt==4.1.2 + # via paramiko +certifi==2024.2.2 + # via requests +cffi==1.16.0 + # via + # cryptography + # pynacl +charset-normalizer==3.3.2 + # via requests +cryptography==42.0.5 + # via paramiko +dnspython==2.6.1 + # via email-validator +email-validator==2.1.1 + # via scitacean h5py==3.10.0 # via scippnexus +idna==3.6 + # via + # email-validator + # requests numpy==1.26.4 # via # h5py # scipp # scipy +paramiko==3.4.0 + # via -r base.in +pycparser==2.22 + # via cffi +pydantic==2.6.4 + # via scitacean +pydantic-core==2.16.3 + # via pydantic +pynacl==1.5.0 + # via paramiko python-dateutil==2.9.0.post0 - # via scippnexus + # via + # scippnexus + # scitacean +requests==2.31.0 + # via scitacean scipp==24.2.0 # via # -r base.in # scippnexus scippnexus==24.3.1 # via -r base.in -scipy==1.12.0 +scipy==1.13.0 # via scippnexus +scitacean==23.10.0 + # via -r base.in six==1.16.0 # via python-dateutil +typing-extensions==4.11.0 + # via + # pydantic + # pydantic-core +urllib3==2.2.1 + # via requests diff --git a/requirements/ci.txt b/requirements/ci.txt index 2f27a0eb..5c27cf8c 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -17,13 +17,13 @@ colorama==0.4.6 # via tox distlib==0.3.8 # via virtualenv -filelock==3.13.1 +filelock==3.13.3 # via # tox # virtualenv gitdb==4.0.11 # via gitpython -gitpython==3.1.42 +gitpython==3.1.43 # via -r ci.in idna==3.6 # via requests @@ -48,7 +48,7 @@ tomli==2.0.1 # via # pyproject-api # tox -tox==4.14.1 +tox==4.14.2 # via -r ci.in urllib3==2.2.1 # via requests diff --git a/requirements/dev.txt b/requirements/dev.txt index c3c5526d..9df9c153 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -12,8 +12,6 @@ -r static.txt -r test.txt -r wheels.txt -annotated-types==0.6.0 - # via pydantic anyio==4.3.0 # via # httpx @@ -26,13 +24,11 @@ arrow==1.3.0 # via isoduration async-lru==2.0.4 # via jupyterlab -cffi==1.16.0 - # via argon2-cffi-bindings click==8.1.7 # via # pip-compile-multi # pip-tools -copier==9.1.1 +copier==9.2.0 # via -r dev.in dunamai==1.19.2 # via copier @@ -42,7 +38,7 @@ funcy==2.0 # via copier h11==0.14.0 # via httpcore -httpcore==1.0.4 +httpcore==1.0.5 # via httpx httpx==0.27.0 # via jupyterlab @@ -50,7 +46,7 @@ isoduration==20.11.0 # via jsonschema jinja2-ansible-filters==1.3.2 # via copier -json5==0.9.22 +json5==0.9.24 # via jupyterlab-server jsonpointer==2.4 # via jsonschema @@ -59,7 +55,7 @@ jsonschema[format-nongpl]==4.21.1 # jupyter-events # jupyterlab-server # nbformat -jupyter-events==0.9.0 +jupyter-events==0.10.0 # via jupyter-server jupyter-lsp==2.2.4 # via jupyterlab @@ -69,11 +65,11 @@ jupyter-server==2.13.0 # jupyterlab # jupyterlab-server # notebook-shim -jupyter-server-terminals==0.5.2 +jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.4 +jupyterlab==4.1.6 # via -r dev.in -jupyterlab-server==2.25.3 +jupyterlab-server==2.26.0 # via jupyterlab notebook-shim==0.2.4 # via jupyterlab @@ -89,16 +85,8 @@ plumbum==1.8.2 # via copier prometheus-client==0.20.0 # via jupyter-server -pycparser==2.21 - # via cffi -pydantic==2.6.3 - # via copier -pydantic-core==2.16.3 - # via pydantic python-json-logger==2.0.7 # via jupyter-events -pyyaml-include==1.3.2 - # via copier questionary==1.10.0 # via copier rfc3339-validator==0.1.4 @@ -109,19 +97,19 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -send2trash==1.8.2 +send2trash==1.8.3 # via jupyter-server sniffio==1.3.1 # via # anyio # httpx -terminado==0.18.0 +terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals toposort==1.10 # via pip-compile-multi -types-python-dateutil==2.8.19.20240311 +types-python-dateutil==2.9.0.20240316 # via arrow uri-template==1.3.0 # via jsonschema @@ -129,7 +117,7 @@ webcolors==1.13 # via jsonschema websocket-client==1.7.0 # via jupyter-server -wheel==0.42.0 +wheel==0.43.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/docs.txt b/requirements/docs.txt index a6caed92..62c2e335 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -26,11 +26,7 @@ beautifulsoup4==4.12.3 # pydata-sphinx-theme bleach==6.1.0 # via nbconvert -certifi==2024.2.2 - # via requests -charset-normalizer==3.3.2 - # via requests -comm==0.2.1 +comm==0.2.2 # via ipykernel debugpy==1.8.1 # via ipykernel @@ -50,13 +46,11 @@ executing==2.0.1 # via stack-data fastjsonschema==2.19.1 # via nbformat -idna==3.6 - # via requests imagesize==1.4.1 # via sphinx -ipykernel==6.29.3 +ipykernel==6.29.4 # via -r docs.in -ipython==8.22.2 +ipython==8.23.0 # via # -r docs.in # ipykernel @@ -72,11 +66,11 @@ jsonschema==4.21.1 # via nbformat jsonschema-specifications==2023.12.1 # via jsonschema -jupyter-client==8.6.0 +jupyter-client==8.6.1 # via # ipykernel # nbclient -jupyter-core==5.7.1 +jupyter-core==5.7.2 # via # ipykernel # jupyter-client @@ -105,11 +99,11 @@ mistune==3.0.2 # via nbconvert myst-parser==2.0.0 # via -r docs.in -nbclient==0.9.0 +nbclient==0.10.0 # via nbconvert -nbconvert==7.16.2 +nbconvert==7.16.3 # via nbsphinx -nbformat==5.9.2 +nbformat==5.10.4 # via # nbclient # nbconvert @@ -126,7 +120,7 @@ packaging==24.0 # sphinx pandocfilters==1.5.1 # via nbconvert -parso==0.8.3 +parso==0.8.4 # via jedi pexpect==4.9.0 # via ipython @@ -155,12 +149,10 @@ pyzmq==25.1.2 # via # ipykernel # jupyter-client -referencing==0.33.0 +referencing==0.34.0 # via # jsonschema # jsonschema-specifications -requests==2.31.0 - # via sphinx rpds-py==0.18.0 # via # jsonschema @@ -204,7 +196,7 @@ tornado==6.4 # via # ipykernel # jupyter-client -traitlets==5.14.1 +traitlets==5.14.2 # via # comm # ipykernel @@ -216,10 +208,6 @@ traitlets==5.14.1 # nbconvert # nbformat # nbsphinx -typing-extensions==4.10.0 - # via pydata-sphinx-theme -urllib3==2.2.1 - # via requests wcwidth==0.2.13 # via prompt-toolkit webencodings==0.5.1 diff --git a/requirements/mypy.txt b/requirements/mypy.txt index 066d33ac..bf11262f 100644 --- a/requirements/mypy.txt +++ b/requirements/mypy.txt @@ -10,5 +10,3 @@ mypy==1.9.0 # via -r mypy.in mypy-extensions==1.0.0 # via mypy -typing-extensions==4.10.0 - # via mypy diff --git a/requirements/nightly.in b/requirements/nightly.in index 0e1f5905..bc75b63b 100644 --- a/requirements/nightly.in +++ b/requirements/nightly.in @@ -3,3 +3,5 @@ # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! scipp >= 24.02.0 scippnexus >= 24.03.0 +scitacean +paramiko diff --git a/requirements/nightly.txt b/requirements/nightly.txt index 02ab46cf..5103bff8 100644 --- a/requirements/nightly.txt +++ b/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:9bb7ade09fe2af7ab62c586f5f5dc6f3e9b8344b +# SHA1:e451bf52ffde344e01fe314804daba72a46b6841 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -6,17 +6,61 @@ # pip-compile-multi # -r basetest.txt +annotated-types==0.6.0 + # via pydantic +bcrypt==4.1.2 + # via paramiko +certifi==2024.2.2 + # via requests +cffi==1.16.0 + # via + # cryptography + # pynacl +charset-normalizer==3.3.2 + # via requests +cryptography==42.0.5 + # via paramiko +dnspython==2.6.1 + # via email-validator +email-validator==2.1.1 + # via scitacean h5py==3.10.0 # via scippnexus +idna==3.6 + # via + # email-validator + # requests +paramiko==3.4.0 + # via -r nightly.in +pycparser==2.22 + # via cffi +pydantic==2.6.4 + # via scitacean +pydantic-core==2.16.3 + # via pydantic +pynacl==1.5.0 + # via paramiko python-dateutil==2.9.0.post0 - # via scippnexus + # via + # scippnexus + # scitacean +requests==2.31.0 + # via scitacean scipp==24.2.0 # via # -r nightly.in # scippnexus scippnexus==24.3.1 # via -r nightly.in -scipy==1.12.0 +scipy==1.13.0 # via scippnexus +scitacean==23.10.0 + # via -r nightly.in six==1.16.0 # via python-dateutil +typing-extensions==4.11.0 + # via + # pydantic + # pydantic-core +urllib3==2.2.1 + # via requests diff --git a/requirements/static.txt b/requirements/static.txt index 0e3af8ac..bc4d3df7 100644 --- a/requirements/static.txt +++ b/requirements/static.txt @@ -9,7 +9,7 @@ cfgv==3.4.0 # via pre-commit distlib==0.3.8 # via virtualenv -filelock==3.13.1 +filelock==3.13.3 # via virtualenv identify==2.5.35 # via pre-commit @@ -17,7 +17,7 @@ nodeenv==1.8.0 # via pre-commit platformdirs==4.2.0 # via virtualenv -pre-commit==3.6.2 +pre-commit==3.7.0 # via -r static.in pyyaml==6.0.1 # via pre-commit diff --git a/requirements/test.in b/requirements/test.in index 7b409792..468b0b9a 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -2,3 +2,5 @@ -r base.in -r basetest.in +scitacean[sftp, test] +pyfakefs diff --git a/requirements/test.txt b/requirements/test.txt index 3c7454d8..1f5c1eba 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,4 +1,4 @@ -# SHA1:ef2ee9576d8a9e65b44e2865a26887eed3fc49d1 +# SHA1:bd856f3c5510027cc5a34a7cdd0a31f05780891d # # This file is autogenerated by pip-compile-multi # To update, run: @@ -7,3 +7,19 @@ # -r base.txt -r basetest.txt +attrs==23.2.0 + # via hypothesis +filelock==3.13.3 + # via scitacean +hypothesis==6.100.1 + # via scitacean +pyfakefs==5.4.0 + # via -r test.in +pyyaml==6.0.1 + # via scitacean +scitacean[sftp,test]==23.10.0 + # via + # -r base.in + # -r test.in +sortedcontainers==2.4.0 + # via hypothesis diff --git a/requirements/wheels.txt b/requirements/wheels.txt index 23d6d310..ff60a184 100644 --- a/requirements/wheels.txt +++ b/requirements/wheels.txt @@ -5,7 +5,7 @@ # # pip-compile-multi # -build==1.1.1 +build==1.2.1 # via -r wheels.in packaging==24.0 # via build diff --git a/src/ess/reduce/scicat.py b/src/ess/reduce/scicat.py new file mode 100644 index 00000000..95387ee6 --- /dev/null +++ b/src/ess/reduce/scicat.py @@ -0,0 +1,37 @@ +from pathlib import Path +from threading import Lock +from typing import Optional + +from scitacean import Client, Dataset + +from .nexus import FilePath + +_file_download_locks = {} + + +def download_scicat_file( + client: Client, + dataset_id: str, + filename: str, + *, + target: Optional[Path] = None, +) -> FilePath: + if target is None: + target = Path(f'~/.cache/essreduce/{dataset_id}') + key = (dataset_id, filename, target) + with _file_download_locks.setdefault(key, Lock()): + dset = client.get_dataset(dataset_id) + dset = client.download_files(dset, target=target, select=filename) + _file_download_locks.pop(key) + return dset.files[0].local_path + + +def get_related_dataset(client: Client, ds: Dataset, relationship: str) -> Dataset: + '''Goes through the datasets related to 'ds' + and finds the one with the selected relation''' + for d in getattr(ds, 'relationships', ()): + if d.relationship == relationship: + return client.get_dataset(d.pid) + raise ValueError( + f'The requested relation "{relationship}" was not found in dataset {ds}' + ) diff --git a/tests/scicat_test.py b/tests/scicat_test.py new file mode 100644 index 00000000..c8a3fddd --- /dev/null +++ b/tests/scicat_test.py @@ -0,0 +1,75 @@ +import hashlib +from pathlib import Path +from tempfile import TemporaryDirectory + +import pytest +from dateutil.parser import parse as parse_date +from scitacean import Dataset, DatasetType, RemotePath +from scitacean.model import Relationship +from scitacean.testing.client import FakeClient, ScicatCommError +from scitacean.testing.transfer import FakeFileTransfer + +from ess.reduce.scicat import download_scicat_file, get_related_dataset + + +def _checksum(data: bytes) -> str: + checksum = hashlib.new("md5") + checksum.update(data) + return checksum.hexdigest() + + +@pytest.fixture +def files(): + return { + "file1.dat": b"contents-of-file1", + "log/what-happened.log": b"ERROR Flux is off the scale", + "thaum.dat": b"0 4 2 59 330 2314552", + } + + +@pytest.fixture +def local_dataset(fs, files): + dset = Dataset( + contact_email="p.stibbons@uu.am", + creation_time=parse_date("1995-08-06T14:14:14"), + owner="pstibbons", + owner_group="faculty", + principal_investigator="m.ridcully@uu.am", + source_folder=RemotePath("/src/stibbons/774"), + creation_location='DTU', + type=DatasetType.RAW, + meta={ + "height": {"value": 0.3, "unit": "m"}, + "mass": "hefty", + }, + relationships=[Relationship(pid='123', relationship='background')], + ) + for name, content in files.items(): + path = Path('tmp') / name + fs.create_file(path, contents=content) + dset.add_local_files(path, base_path='tmp') + return dset + + +def test_download_scicat_file(fs, local_dataset): + local_dataset.make_upload_model() + transfer = FakeFileTransfer(fs=fs) + client = FakeClient.without_login(url="https://fake.scicat", file_transfer=transfer) + uploaded = client.upload_new_dataset_now(local_dataset) + with TemporaryDirectory() as dname: + path = download_scicat_file( + client, + uploaded.pid, + uploaded.files[0].remote_path.posix, + target=dname, + ) + assert path == Path(dname) / uploaded.files[0].remote_path.posix + + +def test_get_related_dataset(local_dataset): + client = FakeClient.without_login(url="https://fake.scicat") + # Looking for comm error here because that indicates the dataset was queried for + with pytest.raises(ScicatCommError): + get_related_dataset(client, local_dataset, 'background') + with pytest.raises(ValueError): + get_related_dataset(client, local_dataset, 'reference')