diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 218600d..1c3cf2e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10"] + python-version: ["3.12"] steps: - uses: actions/checkout@v2 diff --git a/dist/clinical_ETL-2.0.0-py3-none-any.whl b/dist/clinical_ETL-2.0.0-py3-none-any.whl index b08dd4a..299e4bf 100644 Binary files a/dist/clinical_ETL-2.0.0-py3-none-any.whl and b/dist/clinical_ETL-2.0.0-py3-none-any.whl differ diff --git a/pyproject.toml b/pyproject.toml index 17564db..732f273 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,12 +6,11 @@ build-backend = "setuptools.build_meta" version = "2.0.0" name = "clinical_ETL" dependencies = [ - "pandas~=1.3.4", + "pandas>=2.1.0", "pytest>=7.2.0", "pyYAML>=5.4.1", "dateparser>=1.1.0", "openpyxl>=3.0.9", - "jsoncomparison>=1.1.0", "requests>=2.29.0", "jsonschema~=4.19.2", "openapi-spec-validator>=0.7.1", diff --git a/requirements.txt b/requirements.txt index 66f186b..f44f69b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,9 @@ -pandas~=1.3.4 -pytest>=6.2.5 -pyYAML>=5.4.1 -dateparser~=1.1.0 +pandas~=2.1.4 +pytest>=7.2.0 +pyYAML>=6.0.1 +dateparser~=1.2.0 openpyxl~=3.0.9 requests~=2.29 jsonschema~=4.19.1 -jsoncomparison~=1.1.0 openapi-spec-validator~=0.7.1 pdoc3>=0.10.0 diff --git a/src/clinical_ETL.egg-info/PKG-INFO b/src/clinical_ETL.egg-info/PKG-INFO index 8f096ba..a351528 100644 --- a/src/clinical_ETL.egg-info/PKG-INFO +++ b/src/clinical_ETL.egg-info/PKG-INFO @@ -6,12 +6,11 @@ Project-URL: Repository, https://github.com/CanDIG/clinical_ETL_code Requires-Python: >=3.10 Description-Content-Type: text/markdown License-File: LICENSE -Requires-Dist: pandas~=1.3.4 +Requires-Dist: pandas>=2.1.0 Requires-Dist: pytest>=7.2.0 Requires-Dist: pyYAML>=5.4.1 Requires-Dist: dateparser>=1.1.0 Requires-Dist: openpyxl>=3.0.9 -Requires-Dist: jsoncomparison>=1.1.0 Requires-Dist: requests>=2.29.0 Requires-Dist: jsonschema~=4.19.2 Requires-Dist: openapi-spec-validator>=0.7.1 @@ -26,8 +25,9 @@ Specifically, this code was designed to convert clinical data for the MOHCCN pro ## Using clinical_etl as a package You can import this module as a package by including the following in your `requirements.txt`: ``` -clinical_etl@git+https://github.com/CanDIG/clinical_ETL_code.git@main +clinical_etl@git+https://github.com/CanDIG/clinical_ETL_code.git@stable ``` +If you need the latest version, you can replace `stable` with `develop`. ## CSVConvert Most of the heavy lifting is done in the [`CSVConvert.py`](CSVConvert.py) script. See sections below for setting up the inputs and running the script. diff --git a/src/clinical_ETL.egg-info/requires.txt b/src/clinical_ETL.egg-info/requires.txt index dbaa4cd..e243be7 100644 --- a/src/clinical_ETL.egg-info/requires.txt +++ b/src/clinical_ETL.egg-info/requires.txt @@ -1,9 +1,8 @@ -pandas~=1.3.4 +pandas>=2.1.0 pytest>=7.2.0 pyYAML>=5.4.1 dateparser>=1.1.0 openpyxl>=3.0.9 -jsoncomparison>=1.1.0 requests>=2.29.0 jsonschema~=4.19.2 openapi-spec-validator>=0.7.1 diff --git a/src/clinical_etl/CSVConvert.py b/src/clinical_etl/CSVConvert.py index 6ac1ecf..91e9f6a 100644 --- a/src/clinical_etl/CSVConvert.py +++ b/src/clinical_etl/CSVConvert.py @@ -316,8 +316,8 @@ def process_data(raw_csv_dfs): print(f"Processing sheet {page}...") df = raw_csv_dfs[page].dropna(axis='index', how='all') \ .dropna(axis='columns', how='all') \ - .applymap(str) \ - .applymap(lambda x: x.strip()) \ + .map(str) \ + .map(lambda x: x.strip()) \ .drop_duplicates() # drop absolutely identical lines # Sort by identifier and then tag any dups