From 8c9a013cff4ae0cad4c0f4d11585f3854a328719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kub=C3=ADk?= Date: Sat, 30 Nov 2024 01:14:33 +0100 Subject: [PATCH 1/3] fix(az): catch HttpResponseError in _check_hns (#487) * fix(az): catch HttpResponseError in _check_hns * chore: update HISTORY.md --- HISTORY.md | 1 + cloudpathlib/azure/azblobclient.py | 36 +++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index b39bee8c..2e379a6b 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -3,6 +3,7 @@ ## Unreleased - Fixed `CloudPath(...) / other` to correctly attempt to fall back on `other`'s `__rtruediv__` implementation, in order to support classes that explicitly support the `/` with a `CloudPath` instance. Previously, this would always raise a `TypeError` if `other` were not a `str` or `PurePosixPath`. (PR [#479](https://github.com/drivendataorg/cloudpathlib/pull/479)) +- Fixed an uncaught exception on Azure Gen2 storage accounts with HNS enabled when used with `DefaultAzureCredential`. (Issue [#486](https://github.com/drivendataorg/cloudpathlib/issues/486)) ## v0.20.0 (2024-10-18) diff --git a/cloudpathlib/azure/azblobclient.py b/cloudpathlib/azure/azblobclient.py index 80a4a92b..acf49908 100644 --- a/cloudpathlib/azure/azblobclient.py +++ b/cloudpathlib/azure/azblobclient.py @@ -1,9 +1,11 @@ from datetime import datetime, timedelta import mimetypes import os +from http import HTTPStatus from pathlib import Path from typing import Any, Callable, Dict, Iterable, Optional, Tuple, Union +from typing_extensions import cast from ..client import Client, register_client_class from ..cloudpath import implementation_registry @@ -13,7 +15,7 @@ try: - from azure.core.exceptions import ResourceNotFoundError + from azure.core.exceptions import HttpResponseError, ResourceNotFoundError from azure.core.credentials import AzureNamedKeyCredential from azure.storage.blob import ( @@ -202,20 +204,32 @@ def _check_hns(self, cloud_path: AzureBlobPath) -> Optional[bool]: try: account_info = self.service_client.get_account_information() # type: ignore self._hns_enabled = account_info.get("is_hns_enabled", False) # type: ignore + + # get_account_information() not supported with this credential; we have to fallback to + # checking if the root directory exists and is a has 'metadata': {'hdi_isfolder': 'true'} except ResourceNotFoundError: - # get_account_information() not supported with this credential; we have to fallback to - # checking if the root directory exists and is a has 'metadata': {'hdi_isfolder': 'true'} - root_dir = self.service_client.get_blob_client( - container=cloud_path.container, blob="/" - ) - self._hns_enabled = ( - root_dir.exists() - and root_dir.get_blob_properties().metadata.get("hdi_isfolder", False) - == "true" - ) + return self._check_hns_root_metadata(cloud_path) + except HttpResponseError as error: + if error.status_code == HTTPStatus.FORBIDDEN: + self._check_hns_root_metadata(cloud_path) + else: + raise return self._hns_enabled + def _check_hns_root_metadata(self, cloud_path: AzureBlobPath) -> bool: + root_dir = self.service_client.get_blob_client( + container=cloud_path.container, blob="/" + ) + + self._hns_enabled = ( + root_dir.exists() + and root_dir.get_blob_properties().metadata.get("hdi_isfolder", False) + == "true" + ) + + return cast(bool, self._hns_enabled) + def _get_metadata( self, cloud_path: AzureBlobPath ) -> Union["BlobProperties", "FileProperties", Dict[str, Any]]: From c6156adee5a93d832de1720a64b91f2d2440a2e4 Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Fri, 29 Nov 2024 16:15:21 -0800 Subject: [PATCH 2/3] format --- cloudpathlib/azure/azblobclient.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cloudpathlib/azure/azblobclient.py b/cloudpathlib/azure/azblobclient.py index acf49908..78de7891 100644 --- a/cloudpathlib/azure/azblobclient.py +++ b/cloudpathlib/azure/azblobclient.py @@ -218,14 +218,11 @@ def _check_hns(self, cloud_path: AzureBlobPath) -> Optional[bool]: return self._hns_enabled def _check_hns_root_metadata(self, cloud_path: AzureBlobPath) -> bool: - root_dir = self.service_client.get_blob_client( - container=cloud_path.container, blob="/" - ) + root_dir = self.service_client.get_blob_client(container=cloud_path.container, blob="/") self._hns_enabled = ( root_dir.exists() - and root_dir.get_blob_properties().metadata.get("hdi_isfolder", False) - == "true" + and root_dir.get_blob_properties().metadata.get("hdi_isfolder", False) == "true" ) return cast(bool, self._hns_enabled) From 912a696bd463e28ad196c278aaccde2820467caa Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Fri, 29 Nov 2024 17:48:53 -0800 Subject: [PATCH 3/3] cast fallback --- cloudpathlib/azure/azblobclient.py | 5 ++++- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cloudpathlib/azure/azblobclient.py b/cloudpathlib/azure/azblobclient.py index 78de7891..15d72fb3 100644 --- a/cloudpathlib/azure/azblobclient.py +++ b/cloudpathlib/azure/azblobclient.py @@ -5,7 +5,10 @@ from pathlib import Path from typing import Any, Callable, Dict, Iterable, Optional, Tuple, Union -from typing_extensions import cast +try: + from typing import cast +except ImportError: + from typing_extensions import cast from ..client import Client, register_client_class from ..cloudpath import implementation_registry diff --git a/pyproject.toml b/pyproject.toml index e18fd610..bc5dffe6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ ] requires-python = ">=3.8" dependencies = [ - "typing_extensions>4 ; python_version < '3.11'", + "typing-extensions>4 ; python_version < '3.11'", ] [project.optional-dependencies]