Merge pull request #47 from CanDIG/daisieh/updates

.github/workflows/test.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.12"]
 
     steps:
       - uses: actions/checkout@v2

pyproject.toml

@@ -6,12 +6,11 @@ build-backend = "setuptools.build_meta"
 version = "2.0.0"
 name = "clinical_ETL"
 dependencies = [
-    "pandas~=1.3.4",
+    "pandas>=2.1.0",
     "pytest>=7.2.0",
     "pyYAML>=5.4.1",
     "dateparser>=1.1.0",
     "openpyxl>=3.0.9",
-    "jsoncomparison>=1.1.0",
     "requests>=2.29.0",
     "jsonschema~=4.19.2",
     "openapi-spec-validator>=0.7.1",

requirements.txt

@@ -1,10 +1,9 @@
-pandas~=1.3.4
-pytest>=6.2.5
-pyYAML>=5.4.1
-dateparser~=1.1.0
+pandas~=2.1.4
+pytest>=7.2.0
+pyYAML>=6.0.1
+dateparser~=1.2.0
 openpyxl~=3.0.9
 requests~=2.29
 jsonschema~=4.19.1
-jsoncomparison~=1.1.0
 openapi-spec-validator~=0.7.1
 pdoc3>=0.10.0

src/clinical_ETL.egg-info/PKG-INFO

@@ -6,12 +6,11 @@ Project-URL: Repository, https://github.com/CanDIG/clinical_ETL_code
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pandas~=1.3.4
+Requires-Dist: pandas>=2.1.0
 Requires-Dist: pytest>=7.2.0
 Requires-Dist: pyYAML>=5.4.1
 Requires-Dist: dateparser>=1.1.0
 Requires-Dist: openpyxl>=3.0.9
-Requires-Dist: jsoncomparison>=1.1.0
 Requires-Dist: requests>=2.29.0
 Requires-Dist: jsonschema~=4.19.2
 Requires-Dist: openapi-spec-validator>=0.7.1
@@ -26,8 +25,9 @@ Specifically, this code was designed to convert clinical data for the MOHCCN pro
 ## Using clinical_etl as a package
 You can import this module as a package by including the following in your `requirements.txt`:
 ```
-clinical_etl@git+https://github.com/CanDIG/clinical_ETL_code.git@main
+clinical_etl@git+https://github.com/CanDIG/clinical_ETL_code.git@stable
 ```
+If you need the latest version, you can replace `stable` with `develop`.
 
 ## CSVConvert
 Most of the heavy lifting is done in the [`CSVConvert.py`](CSVConvert.py) script. See sections below for setting up the inputs and running the script.

src/clinical_ETL.egg-info/requires.txt

@@ -1,9 +1,8 @@
-pandas~=1.3.4
+pandas>=2.1.0
 pytest>=7.2.0
 pyYAML>=5.4.1
 dateparser>=1.1.0
 openpyxl>=3.0.9
-jsoncomparison>=1.1.0
 requests>=2.29.0
 jsonschema~=4.19.2
 openapi-spec-validator>=0.7.1

src/clinical_etl/CSVConvert.py

@@ -316,8 +316,8 @@ def process_data(raw_csv_dfs):
         print(f"Processing sheet {page}...")
         df = raw_csv_dfs[page].dropna(axis='index', how='all') \
             .dropna(axis='columns', how='all') \
-            .applymap(str) \
-            .applymap(lambda x: x.strip()) \
+            .map(str) \
+            .map(lambda x: x.strip()) \
             .drop_duplicates()  # drop absolutely identical lines
 
         # Sort by identifier and then tag any dups