diff --git a/Pipfile b/Pipfile index 17d8c6af..a8433381 100644 --- a/Pipfile +++ b/Pipfile @@ -32,7 +32,7 @@ pysftp = "~=0.2" python-dotenv = "~=1.0" requests = "~=2.28" slackclient = "~=2.9" -sqlalchemy = "~=1.4" +sqlalchemy = "~=2.0" lab-share-lib = { git = 'https://github.com/sanger/lab-share-lib.git', ref = 'master', editable = false } [requires] diff --git a/Pipfile.lock b/Pipfile.lock index 05b05fab..965c224c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "922a689def1350236b896a77999fe8e462a7337d7284c14ff2630bc215591baf" + "sha256": "3a309f301bb9ccfb3c763ac6bc0d9378810d8e319d6aba1c08c5718e4739cd07" }, "pipfile-spec": 6, "requires": { @@ -592,7 +592,7 @@ "sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1", "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526" ], - "markers": "python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))", + "markers": "platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))", "version": "==2.0.2" }, "gunicorn": { @@ -1148,50 +1148,58 @@ }, "sqlalchemy": { "hashes": [ - "sha256:07e48cbcdda6b8bc7a59d6728bd3f5f574ffe03f2c9fb384239f3789c2d95c2e", - "sha256:18cafdb27834fa03569d29f571df7115812a0e59fd6a3a03ccb0d33678ec8420", - "sha256:1b1e5e96e2789d89f023d080bee432e2fef64d95857969e70d3cadec80bd26f0", - "sha256:315676344e3558f1f80d02535f410e80ea4e8fddba31ec78fe390eff5fb8f466", - "sha256:31de1e2c45e67a5ec1ecca6ec26aefc299dd5151e355eb5199cd9516b57340be", - "sha256:3d94682732d1a0def5672471ba42a29ff5e21bb0aae0afa00bb10796fc1e28dd", - "sha256:3ec187acf85984263299a3f15c34a6c0671f83565d86d10f43ace49881a82718", - "sha256:4847f4b1d822754e35707db913396a29d874ee77b9c3c3ef3f04d5a9a6209618", - "sha256:4d112b0f3c1bc5ff70554a97344625ef621c1bfe02a73c5d97cac91f8cd7a41e", - "sha256:51e1ba2884c6a2b8e19109dc08c71c49530006c1084156ecadfaadf5f9b8b053", - "sha256:535377e9b10aff5a045e3d9ada8a62d02058b422c0504ebdcf07930599890eb0", - "sha256:5dbf17ac9a61e7a3f1c7ca47237aac93cabd7f08ad92ac5b96d6f8dea4287fc1", - "sha256:5f752676fc126edc1c4af0ec2e4d2adca48ddfae5de46bb40adbd3f903eb2120", - "sha256:64cb0ad8a190bc22d2112001cfecdec45baffdf41871de777239da6a28ed74b6", - "sha256:6913b8247d8a292ef8315162a51931e2b40ce91681f1b6f18f697045200c4a30", - "sha256:69fac0a7054d86b997af12dc23f581cf0b25fb1c7d1fed43257dee3af32d3d6d", - "sha256:7001f16a9a8e06488c3c7154827c48455d1c1507d7228d43e781afbc8ceccf6d", - "sha256:7b81b1030c42b003fc10ddd17825571603117f848814a344d305262d370e7c34", - "sha256:7f8267682eb41a0584cf66d8a697fef64b53281d01c93a503e1344197f2e01fe", - "sha256:887865924c3d6e9a473dc82b70977395301533b3030d0f020c38fd9eba5419f2", - "sha256:9167d4227b56591a4cc5524f1b79ccd7ea994f36e4c648ab42ca995d28ebbb96", - "sha256:939f9a018d2ad04036746e15d119c0428b1e557470361aa798e6e7d7f5875be0", - "sha256:955162ad1a931fe416eded6bb144ba891ccbf9b2e49dc7ded39274dd9c5affc5", - "sha256:984ee13543a346324319a1fb72b698e521506f6f22dc37d7752a329e9cd00a32", - "sha256:9883f5fae4fd8e3f875adc2add69f8b945625811689a6c65866a35ee9c0aea23", - "sha256:a1ad90c97029cc3ab4ffd57443a20fac21d2ec3c89532b084b073b3feb5abff3", - "sha256:a3714e5b33226131ac0da60d18995a102a17dddd42368b7bdd206737297823ad", - "sha256:ae067ab639fa499f67ded52f5bc8e084f045d10b5ac7bb928ae4ca2b6c0429a5", - "sha256:b33ffbdbbf5446cf36cd4cc530c9d9905d3c2fe56ed09e25c22c850cdb9fac92", - "sha256:b6e4cb5c63f705c9d546a054c60d326cbde7421421e2d2565ce3e2eee4e1a01f", - "sha256:b7f4b6aa6e87991ec7ce0e769689a977776db6704947e562102431474799a857", - "sha256:c04144a24103135ea0315d459431ac196fe96f55d3213bfd6d39d0247775c854", - "sha256:c522e496f9b9b70296a7675272ec21937ccfc15da664b74b9f58d98a641ce1b6", - "sha256:c5a99282848b6cae0056b85da17392a26b2d39178394fc25700bcf967e06e97a", - "sha256:c7a46639ba058d320c9f53a81db38119a74b8a7a1884df44d09fbe807d028aaf", - "sha256:d4b1cc7835b39835c75cf7c20c926b42e97d074147c902a9ebb7cf2c840dc4e2", - "sha256:d4d164df3d83d204c69f840da30b292ac7dc54285096c6171245b8d7807185aa", - "sha256:d61e9ecc849d8d44d7f80894ecff4abe347136e9d926560b818f6243409f3c86", - "sha256:d68e1762997bfebf9e5cf2a9fd0bcf9ca2fdd8136ce7b24bbd3bbfa4328f3e4a", - "sha256:e3c1808008124850115a3f7e793a975cfa5c8a26ceeeb9ff9cbb4485cac556df", - "sha256:f8cb80fe8d14307e4124f6fad64dfd87ab749c9d275f82b8b4ec84c84ecebdbe" + "sha256:01704ec4a6877b74608264992a87979a27a8927cefd14ccdc0d478acacc1ed85", + "sha256:0186b970fd4561def531b582a86819d8f8af65c8b1a78cf015ee47e526f4cfb6", + "sha256:05b81afdc25d1ce43cb59647c9992559dc7487b1670ccab0426fc8b8f859e933", + "sha256:08c9169692722df8a2ef6c6ff1055e11563c990e9c74df9af62139a0c6397b8c", + "sha256:0fcc9b2f5b334fdaf278459dfc0fb86d3a0317ae8ce813a7a3ef8639b44b6e4a", + "sha256:101df3fa8f207ade1124d7729f6c9eab28a2560baa31b3e131e76a599d884b33", + "sha256:1f504779e6e68d0cb7043825958125abd7742c7c73ce9c6b652d20c6b5f17022", + "sha256:20b9e36f0219285c580dc5e98cadb59b751e259f3829460bc58d45e7a770dd36", + "sha256:31d019c60f4817b24c484d3110c7754cd2b8f7070057eddef5822994bf16da5a", + "sha256:34a4e134eac68354cce40b35ccdbc91ff67ce0c791ea4aa81e021f2ee14bfefb", + "sha256:3846d36c1ca113a7fa078abb5e69a8c3d1c7642baf12267dcd9a0d660cf1bdeb", + "sha256:3997238968fa495fac4b17fa18b36616c41a6e6759f323dfb3f83cbcf1d3b1bb", + "sha256:476bd377f430b1871f058332696ef61c42dfa8ad242ebb8bcf212c3d4127ea8a", + "sha256:54b24a20cca275ada37ba40aa87dd257fda6e7da7f448d3282b6124d940f64d5", + "sha256:5bc451ee18776dcb6b2ac8c154db0536f75a2535f5da055179734f5e7f2e7b72", + "sha256:619784c399f5c4240b002e4dba30cfba15696274614da77846b69b2c9a74066b", + "sha256:655e93fabd11bf53e6af44cee152b608d49ece4b4d9cc29328dd476faaa47c0c", + "sha256:664a21613d7eff895de9ef731632575cfca773ddbac9b7f7adad288ab971bcbd", + "sha256:664ec164bc01ab66dfd19062ca7982a9ea12274596e17732908eb78621adc147", + "sha256:67c35b33a0828b4f5ac6e76a1b6a54b27d693599c93ea7a4c8e53ff52796378f", + "sha256:6dd8405bd1ffcbf11fda0e6b172e7e90044610de16325295efe92367551f666d", + "sha256:70d38432d75f6c95973f9713b30881e40a4e8d8ccfe8bbeb55466d8c737acc79", + "sha256:7b07b83789997cf83ce9a5e7156a2b9a6cb54a4137add8ad95eff32f6746279b", + "sha256:7e23205506a437476dce8193357ce47254cce7c94018b1b4856476ad2e74f1ae", + "sha256:8770318683c8e08976633cec2c9711eb4279553ecbad1ca97f82c5b9174e0e76", + "sha256:8f9085bedb9e2f2bf714cfd86be6deaa7050f998843a3a0e595ec3eb0d25c743", + "sha256:9efb27e899cf7d43cf42c0852ef772a8b568c39dc7b55768a5a80c67bb64dfc2", + "sha256:a5e1826a1ebbbbc26285a0304d7cafff4ec63cdae83fde89d5f2ec67f4444a44", + "sha256:a97c4b5527ea563867ccbee031af93932d9699c6c73f1ea70adcbc935c80379e", + "sha256:aeb49e1436d6558d31c006b385a5071e802be6db257ce36940e66cefce92aa72", + "sha256:c34c6b7975cb9e4848d4366d54a634bbced7b491a36029642c7e738a44b595a3", + "sha256:c681d0f59c8ed12fd3f68d08d423354b1cc501220ddabc7a20b9ca8ed52b8f70", + "sha256:ca2ce5f3125cb6e043c90dd901446b74878f35eb6660e0e58d7ef02832f7d332", + "sha256:d1588f6ba25dbb2d6eb1531e56f419e02cdc9ec06d9f082195877c5148f6f6ab", + "sha256:db3e4db26c1a771d7b23a1031eaf351cfcaaa96d463ae900bb56c6a6f0585fbf", + "sha256:e49e9aefffe9c598a6ccf8d2dbb4556f4d93d0ae346b9d199b3712d24af0ce75", + "sha256:e60bec8fdd753212aa8cec012bbb3060e9c2227496fa935ca8918744a34c864d", + "sha256:eeec87ebe90018bc871b84b03e4bff5dbdc722e28b8f5a6e9a94486eb0cb4902", + "sha256:eff376cc201363634b5b60a828b3998b088a71e16f7a43da26fc0e2201e25a05", + "sha256:f44c37e03cb941dd0db371a9f391cfb586c9966f436bf18b5492ee26f5ac6a5b", + "sha256:f707729cc35dbd1d672b11037f5464b8a42c1e89772d7fc60648da215fa72fc6" ], "index": "pypi", - "version": "==1.4.46" + "version": "==2.0.1" + }, + "typing-extensions": { + "hashes": [ + "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa", + "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e" + ], + "markers": "python_version >= '3.7'", + "version": "==4.4.0" }, "tzdata": { "hashes": [ diff --git a/setup_test_db.py b/setup_test_db.py index a2acd168..cb6b1c9b 100644 --- a/setup_test_db.py +++ b/setup_test_db.py @@ -1,4 +1,6 @@ # flake8: noqa +from sqlalchemy import text + import crawler.config.test as config from crawler.db.mysql import create_mysql_connection_engine @@ -11,6 +13,7 @@ create_db = """ CREATE DATABASE IF NOT EXISTS `unified_warehouse_test` /*!40100 DEFAULT CHARACTER SET latin1 */; """ + drop_table_lh_sample = """ DROP TABLE IF EXISTS `unified_warehouse_test`.`lighthouse_sample`; """ @@ -204,25 +207,25 @@ """ with sql_engine.connect() as connection: - connection.execute(create_db) + connection.execute(text(create_db)) print("*** Dropping table LIGHTHOUSE SAMPLE ***") - connection.execute(drop_table_lh_sample) + connection.execute(text(drop_table_lh_sample)) print("*** Dropping table STOCK RESOURCE ***") - connection.execute(drop_table_stock_resource) + connection.execute(text(drop_table_stock_resource)) print("*** Dropping table STUDY ***") - connection.execute(drop_table_study) + connection.execute(text(drop_table_study)) print("*** Dropping table SAMPLE ***") - connection.execute(drop_table_sample) + connection.execute(text(drop_table_sample)) print("*** Creating table SAMPLE ***") - connection.execute(create_table_sample) + connection.execute(text(create_table_sample)) print("*** Creating table STUDY ***") - connection.execute(create_table_study) + connection.execute(text(create_table_study)) print("*** Creating table STOCK RESOURCE ***") - connection.execute(create_table_stock_resource) + connection.execute(text(create_table_stock_resource)) print("*** Creating table LIGHTHOUSE SAMPLE ***") - connection.execute(create_table_lh_sample) + connection.execute(text(create_table_lh_sample)) print("Initialising the test MySQL events warehouse database") @@ -370,36 +373,36 @@ """ with sql_engine.connect() as connection: - connection.execute(create_db) + connection.execute(text(create_db)) print("*** Dropping view CHERRYPICKED SAMPLES ***") - connection.execute(drop_view_cherrypicked_samples) + connection.execute(text(drop_view_cherrypicked_samples)) print("*** Dropping table ROLES ***") - connection.execute(drop_table_roles) + connection.execute(text(drop_table_roles)) print("*** Dropping table ROLE TYPES ***") - connection.execute(drop_table_role_types) + connection.execute(text(drop_table_role_types)) print("*** Dropping table EVENTS ***") - connection.execute(drop_table_events) + connection.execute(text(drop_table_events)) print("*** Dropping table EVENT TYPES ***") - connection.execute(drop_table_event_types) + connection.execute(text(drop_table_event_types)) print("*** Dropping table SUBJECT ***") - connection.execute(drop_table_subjects) + connection.execute(text(drop_table_subjects)) print("*** Dropping table SUBJECT TYPES ***") - connection.execute(drop_table_subject_types) + connection.execute(text(drop_table_subject_types)) print("*** Creating table SUBJECT TYPES ***") - connection.execute(create_table_subject_types) + connection.execute(text(create_table_subject_types)) print("*** Creating table SUBJECTS ***") - connection.execute(create_table_subjects) + connection.execute(text(create_table_subjects)) print("*** Creating table EVENT TYPES ***") - connection.execute(create_table_event_types) + connection.execute(text(create_table_event_types)) print("*** Creating table EVENTS ***") - connection.execute(create_table_events) + connection.execute(text(create_table_events)) print("*** Creating table ROLE TYPES ***") - connection.execute(create_table_role_types) + connection.execute(text(create_table_role_types)) print("*** Creating table ROLES ***") - connection.execute(create_table_roles) + connection.execute(text(create_table_roles)) print("*** Creating view CHERRYPICKED SAMPLES ***") - connection.execute(create_cherrypicked_samples_view) + connection.execute(text(create_cherrypicked_samples_view)) print("Done") diff --git a/tests/conftest.py b/tests/conftest.py index 42fb5dc3..ba56feb2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -442,8 +442,8 @@ def delete_sql_engine_tables(engine, tables): def get_table(sql_engine, table_name): - metadata = MetaData(sql_engine) - metadata.reflect() + metadata = MetaData() + metadata.reflect(sql_engine) return metadata.tables[table_name] diff --git a/tests/migrations/test_back_populate_source_plate_and_sample_uuids.py b/tests/migrations/test_back_populate_source_plate_and_sample_uuids.py index d17c5e40..5970ecb3 100644 --- a/tests/migrations/test_back_populate_source_plate_and_sample_uuids.py +++ b/tests/migrations/test_back_populate_source_plate_and_sample_uuids.py @@ -2,6 +2,7 @@ import pytest from pymongo import ASCENDING +from sqlalchemy import text from crawler.constants import ( FIELD_LH_SAMPLE_UUID, @@ -83,23 +84,23 @@ def test_back_populate_source_plate_uuid_and_sample_uuid_populates_sample_uuid( viewed_uuids.append(sample[FIELD_LH_SAMPLE_UUID]) # Now we check in mlwh - cursor = query_lighthouse_sample.execute( - "SELECT COUNT(*) FROM lighthouse_sample WHERE plate_barcode = '123' AND lh_sample_uuid IS NOT NULL" + cursor_result = query_lighthouse_sample.execute( + text("SELECT COUNT(*) FROM lighthouse_sample WHERE plate_barcode = '123' AND lh_sample_uuid IS NOT NULL") ) - sample_count = cursor.fetchone()[0] + sample_count = cursor_result.first()[0] assert sample_count == len(samples_after) - cursor = query_lighthouse_sample.execute( - "SELECT * FROM lighthouse_sample WHERE plate_barcode = '123' AND lh_sample_uuid IS NOT NULL" + cursor_result = query_lighthouse_sample.execute( + text("SELECT * FROM lighthouse_sample WHERE plate_barcode = '123' AND lh_sample_uuid IS NOT NULL") ) - obtained_mlwh_samples = list(cursor.fetchall()) + results = list(cursor_result.mappings()) mongo_dict = {} for sample in samples_after: mongo_dict[str(sample[FIELD_MONGODB_ID])] = sample[FIELD_LH_SAMPLE_UUID] mlwh_dict = {} - for mlsample in obtained_mlwh_samples: + for mlsample in results: mlwh_dict[mlsample[MLWH_MONGODB_ID]] = mlsample[FIELD_LH_SAMPLE_UUID] for mongo_id in mongo_dict.keys(): @@ -157,23 +158,23 @@ def test_back_populate_source_plate_uuid_and_sample_uuid_has_source_plate_uuid( assert sample[FIELD_LH_SOURCE_PLATE_UUID] == source_plate_uuid # Now we check in mlwh - cursor = query_lighthouse_sample.execute( - "SELECT COUNT(*) FROM lighthouse_sample WHERE lh_source_plate_uuid IS NOT NULL" + cursor_result = query_lighthouse_sample.execute( + text("SELECT COUNT(*) FROM lighthouse_sample WHERE lh_source_plate_uuid IS NOT NULL") ) - sample_count = cursor.fetchone()[0] + sample_count = cursor_result.first()[0] assert sample_count == len(samples_after) - cursor = query_lighthouse_sample.execute( - "SELECT * FROM lighthouse_sample WHERE lh_source_plate_uuid IS NOT NULL ORDER BY mongodb_id ASC" + cursor_result = query_lighthouse_sample.execute( + text("SELECT * FROM lighthouse_sample WHERE lh_source_plate_uuid IS NOT NULL ORDER BY mongodb_id ASC") ) - obtained_mlwh_samples = list(cursor.fetchall()) + results = list(cursor_result.mappings()) mongo_dict = {} for sample in samples_after: mongo_dict[str(sample[FIELD_MONGODB_ID])] = sample[FIELD_LH_SOURCE_PLATE_UUID] mlwh_dict = {} - for mlsample in obtained_mlwh_samples: + for mlsample in results: mlwh_dict[mlsample[MLWH_MONGODB_ID]] = mlsample[FIELD_LH_SOURCE_PLATE_UUID] for mongo_id in mongo_dict.keys(): diff --git a/tests/migrations/test_reconnect_mlwh_with_mongo.py b/tests/migrations/test_reconnect_mlwh_with_mongo.py index d8bfabcf..25bf5303 100644 --- a/tests/migrations/test_reconnect_mlwh_with_mongo.py +++ b/tests/migrations/test_reconnect_mlwh_with_mongo.py @@ -1,4 +1,5 @@ import pytest +from sqlalchemy import text from crawler.constants import FIELD_MONGODB_ID, FIELD_PLATE_BARCODE, FIELD_RNA_ID, MLWH_MONGODB_ID, MLWH_RNA_ID from migrations import reconnect_mlwh_with_mongo @@ -35,15 +36,15 @@ def test_reconnect_mlwh_with_mongo_can_connect_with_mlwh( reconnect_mlwh_with_mongo.run(config, filepath) # Now we check in mlwh - cursor = query_lighthouse_sample.execute( - "SELECT rna_id, mongodb_id FROM lighthouse_sample WHERE plate_barcode = '123'" + cursor_result = query_lighthouse_sample.execute( + text("SELECT rna_id, mongodb_id FROM lighthouse_sample WHERE plate_barcode = '123'") ) + results = list(cursor_result.mappings()) - obtained_mlwh_samples = list(cursor.fetchall()) - assert obtained_mlwh_samples[0]["mongodb_id"] == str(samples_in_mongo[0][FIELD_MONGODB_ID]) + assert results[0]["mongodb_id"] == str(samples_in_mongo[0][FIELD_MONGODB_ID]) mlwh_dict = {} - for mlsample in obtained_mlwh_samples: + for mlsample in results: mlwh_dict[mlsample[MLWH_RNA_ID]] = mlsample[MLWH_MONGODB_ID] mongo_dict = {}