add url stuff to the mock test client

tests/mock_clients/mock_azureblob.py

@@ -6,6 +6,7 @@
 
 
 from azure.storage.blob import BlobProperties
+from azure.storage.blob._shared.authentication import SharedKeyCredentialPolicy
 from azure.core.exceptions import ResourceNotFoundError
 
 from .utils import delete_empty_parents_up_to_root
@@ -30,6 +31,20 @@ def __init__(self, *args, **kwargs):
         def from_connection_string(cls, *args, **kwargs):
             return cls()
 
+        @property
+        def account_name(self) -> str:
+            """Returns well-known account name used by Azurite
+            See: https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio%2Cblob-storage#well-known-storage-account-and-key
+            """
+            return "devstoreaccount1"
+
+        @property
+        def credential(self):
+            """Returns well-known account key used by Azurite
+            See: https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio%2Cblob-storage#well-known-storage-account-and-key
+            """
+            return SharedKeyCredentialPolicy(self.account_name, "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==")
+
         def __del__(self):
             self.tmp.cleanup()
 

tests/mock_clients/mock_gs.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 from pathlib import Path, PurePosixPath
 import shutil
 from tempfile import TemporaryDirectory
@@ -38,6 +38,9 @@ def bucket(self, bucket):
         def list_buckets(self):
             return [DEFAULT_GS_BUCKET_NAME]
 
+        def get_bucket(self, bucket):
+            return MockBucket(self.tmp_path, bucket, client=self)
+
     return MockClient
 
 
@@ -106,6 +109,13 @@ def updated(self):
     def content_type(self):
         return self.client.metadata_cache.get(self.bucket / self.name, None)
 
+    @property
+    def public_url(self) -> str:
+        return f"https://storage.googleapis.com{self.bucket}/{self.name}"
+
+    def generate_signed_url(self, version: str, expiration: timedelta, method: str):
+        return f"https://storage.googleapis.com{self.bucket}/{self.name}?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=TEST&X-Goog-Date=20240131T185515Z&X-Goog-Expires=3600&X-Goog-SignedHeaders=host&X-Goog-Signature=TEST"
+
 
 class MockBucket:
     def __init__(self, name, bucket_name, client=None):
@@ -149,7 +159,6 @@ def list_blobs(self, max_results=None, prefix=None, delimiter=None):
                 f"Bucket {self.name} not expected as mock bucket; only '{DEFAULT_GS_BUCKET_NAME}' exists."
             )
 
-
 class MockHTTPIterator:
     def __init__(self, blobs, sub_directories, max_results):
         self.blobs = blobs

tests/mock_clients/mock_s3.py

@@ -231,6 +231,10 @@ def head_object(self, Bucket, Key, **kwargs):
         else:
             return {"key": Key}
 
+    def generate_presigned_url(self, op: str, Params: dict, ExpiresIn: int):
+        mock_presigned_url = f"https://{Params['Bucket']}.s3.amazonaws.com/{Params['Key']}?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=TEST%2FTEST%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240131T194721Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=TEST"
+        return mock_presigned_url
+
     @property
     def exceptions(self):
         Ex = collections.namedtuple("Ex", "NoSuchKey")

tests/test_cloudpath_file_io.py

@@ -475,3 +475,8 @@ def test_drive_exists(rig):
     assert CloudPath(f"{rig.cloud_prefix}{p.drive}").exists()
 
     assert not CloudPath(f"{rig.cloud_prefix}totally-fake-not-existing-bucket-for-tests").exists()
+
+
+def test_as_url(rig):
+    p: CloudPath = rig.create_cloud_path("dir_0/file0_0.txt")
+    presigned_url = p.as_url(presign=True)

tests/test_s3_specific.py

@@ -247,3 +247,17 @@ def test_aws_endpoint_url_env(monkeypatch):
     monkeypatch.setenv("AWS_ENDPOINT_URL", localstack_url)
     s3_client_custom_endpoint = S3Client()
     assert s3_client_custom_endpoint.client.meta.endpoint_url == localstack_url
+
+
+def test_as_url(monkeypatch):
+    path = S3Path("s3://arxiv/pdf")
+    public_url = path.as_url()
+    assert public_url == f"https://arxiv.s3.amazonaws.com/pdf"
+
+    localstack_url = "http://localhost:4566"
+    monkeypatch.setenv("AWS_ENDPOINT_URL", localstack_url)
+    s3_client_custom_endpoint = S3Client()
+
+    path = S3Path("s3://arxiv/pdf", client=s3_client_custom_endpoint)
+    public_url = path.as_url()
+    assert public_url == f"{localstack_url}/arxiv/pdf"