diff --git a/setup.cfg b/setup.cfg index 1a19589..3b851e7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -59,7 +59,7 @@ install_requires = requests python-dateutil python-dotenv - univers == 30.11.0 + univers >= 30.11.0 [options.packages.find] diff --git a/src/fetchcode/package.py b/src/fetchcode/package.py index b9035c8..f5d070e 100644 --- a/src/fetchcode/package.py +++ b/src/fetchcode/package.py @@ -15,8 +15,6 @@ # specific language governing permissions and limitations under the License. import dataclasses -import logging -import os import re import time from typing import List @@ -36,17 +34,12 @@ from fetchcode.package_util import OpenSSLGitHubSource from fetchcode.package_util import construct_cocoapods_package from fetchcode.package_util import get_cocoapod_tags -from fetchcode.package_util import get_cocoapods_org_url_status -from fetchcode.package_util import get_pod_data_with_soup from fetchcode.packagedcode_models import Package from fetchcode.utils import get_hashed_path from fetchcode.utils import get_response router = Router() -LOG_FILE_LOCATION = os.path.join(os.path.expanduser("~"), "purlcli.log") -logger = logging.getLogger(__name__) - def info(url): """ @@ -374,113 +367,40 @@ def get_gnu_data_from_purl(purl): @router.route("pkg:cocoapods/.*") def get_cocoapods_data_from_purl(purl): - """ - Generate `Package` object from the `purl` string of cocoapods type - """ - logging.basicConfig( - filename=LOG_FILE_LOCATION, - level=logging.WARN, - format="%(levelname)s - %(message)s", - filemode="w", - ) - purl = PackageURL.from_string(purl) name = purl.name - version = purl.version cocoapods_org_url = f"https://cocoapods.org/pods/{name}" - repository_homepage_url = f"https://cocoapods.org/pods/{name}" - - purl_to_cocoapods_org_url_status = get_cocoapods_org_url_status(purl, name, cocoapods_org_url) - cocoa_org_url_status = purl_to_cocoapods_org_url_status["return_message"] - - status_values = [ - "cocoapods_org_redirects_to_github", - "cocoapods_org_url_redirects", - "failed_to_fetch_github_redirect", - "github_redirect_error", - "github_redirect_not_found", - ] - - cocoa_org_url_status_code = None - if cocoa_org_url_status == "cocoapods_org_url_not_found": - cocoa_org_url_status_code = 404 - elif cocoa_org_url_status == "cocoapods_org_url_temporarily_unavailable": - cocoa_org_url_status_code = 503 - elif any(cocoa_org_url_status == status for status in status_values): - cocoa_org_url_status_code = 302 - - if ( - cocoa_org_url_status == "cocoapods_org_url_not_found" - or cocoa_org_url_status == "cocoapods_org_url_redirects" - or cocoa_org_url_status == "cocoapods_org_url_temporarily_unavailable" - or cocoa_org_url_status == "failed_to_fetch_github_redirect" - or cocoa_org_url_status == "github_redirect_error" - or cocoa_org_url_status == "github_redirect_not_found" - ): - return - - purl_to_pod_data_with_soup = {} - if cocoa_org_url_status_code != 302 and cocoa_org_url_status_code != 503: - purl_to_pod_data_with_soup = get_pod_data_with_soup(purl, name, cocoapods_org_url) - - cocoapods_org_pod_name = None - if purl_to_pod_data_with_soup.get('cocoapods_org_pod_name') is not None: - cocoapods_org_pod_name = purl_to_pod_data_with_soup["cocoapods_org_pod_name"] - elif purl_to_cocoapods_org_url_status.get('cocoapods_org_pod_name') is not None: - cocoapods_org_pod_name = purl_to_cocoapods_org_url_status["cocoapods_org_pod_name"] - - cocoapods_org_version = None - cocoapods_org_gh_repo_owner = None - cocoapods_org_gh_repo_name = None - - if purl_to_pod_data_with_soup.get("cocoapods_org_version") is not None: - cocoapods_org_version = purl_to_pod_data_with_soup["cocoapods_org_version"] - elif purl_to_cocoapods_org_url_status.get("cocoapods_org_version") is not None: - cocoapods_org_version = purl_to_cocoapods_org_url_status[ - "cocoapods_org_version" - ] - - if purl_to_pod_data_with_soup.get("cocoapods_org_gh_repo_owner") is not None: - cocoapods_org_gh_repo_owner = purl_to_pod_data_with_soup[ - "cocoapods_org_gh_repo_owner" - ] - elif ( - purl_to_cocoapods_org_url_status.get("cocoapods_org_gh_repo_owner") is not None - ): - cocoapods_org_gh_repo_owner = purl_to_cocoapods_org_url_status[ - "cocoapods_org_gh_repo_owner" - ] - - if purl_to_pod_data_with_soup.get("cocoapods_org_gh_repo_name") is not None: - cocoapods_org_gh_repo_name = purl_to_pod_data_with_soup[ - "cocoapods_org_gh_repo_name" - ] - elif purl_to_cocoapods_org_url_status.get("cocoapods_org_gh_repo_name") is not None: - cocoapods_org_gh_repo_name = purl_to_cocoapods_org_url_status[ - "cocoapods_org_gh_repo_name" - ] - api = "https://cdn.cocoapods.org" - hashed_path = get_hashed_path(cocoapods_org_pod_name) + hashed_path = get_hashed_path(name) hashed_path_underscore = hashed_path.replace("/", "_") file_prefix = "all_pods_versions_" spec = f"{api}/{file_prefix}{hashed_path_underscore}.txt" - data_list = get_cocoapod_tags(spec, cocoapods_org_pod_name) - if not version: - version = cocoapods_org_version + data_list = get_cocoapod_tags(spec, name) + for tag in data_list: if purl.version and tag != purl.version: continue + gh_repo_owner = None + gh_repo_name = name + podspec_api_url = f"https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/{hashed_path}/{name}/{tag}/{name}.podspec.json" + podspec_api_response = get_response(podspec_api_url) + podspec_homepage = podspec_api_response.get('homepage') + + if podspec_homepage.startswith("https://github.com/"): + podspec_homepage_remove_gh_prefix = podspec_homepage.replace("https://github.com/", "") + podspec_homepage_split = podspec_homepage_remove_gh_prefix.split("/") + gh_repo_owner = podspec_homepage_split[0] + gh_repo_name = podspec_homepage_split[-1] + tag_pkg = construct_cocoapods_package( purl, name, hashed_path, - repository_homepage_url, - cocoapods_org_gh_repo_owner, - cocoapods_org_gh_repo_name, - tag, - cocoapods_org_pod_name + cocoapods_org_url, + gh_repo_owner, + gh_repo_name, + tag ) yield tag_pkg diff --git a/src/fetchcode/package_util.py b/src/fetchcode/package_util.py index abeaba5..33b06d9 100644 --- a/src/fetchcode/package_util.py +++ b/src/fetchcode/package_util.py @@ -15,20 +15,13 @@ # specific language governing permissions and limitations under the License. import dataclasses -import logging -import os import re import attr -from bs4 import BeautifulSoup -from univers import versions from fetchcode import utils from fetchcode.packagedcode_models import Package -LOG_FILE_LOCATION = os.path.join(os.path.expanduser("~"), "purlcli.log") -logger = logging.getLogger(__name__) - def package_from_dict(package_data): """ @@ -732,198 +725,19 @@ def get_package_info(cls, gh_purl, package_name): } -def get_cocoapods_org_url_status(purl, name, cocoapods_org_url): - purl_to_cocoapods_org_url_status = {} - cocoapods_org_url_head_request = utils.make_head_request(cocoapods_org_url) - cocoapods_org_url_status_code = cocoapods_org_url_head_request.status_code - - if cocoapods_org_url_status_code == 404: - logger.error(f"cocoapods_org_url not found for {name}") - purl_to_cocoapods_org_url_status["return_message"] = "cocoapods_org_url_not_found" - return purl_to_cocoapods_org_url_status - elif cocoapods_org_url_status_code == 503: - logger.error(f"cocoapods_org_url temporarily unavailable for {name}") - purl_to_cocoapods_org_url_status["return_message"] = "cocoapods_org_url_temporarily_unavailable" - return purl_to_cocoapods_org_url_status - elif cocoapods_org_url_status_code == 302: - redirect_url = cocoapods_org_url_head_request.headers['Location'] - redirect_message = f"The cocoapods.org URL {cocoapods_org_url} redirects to {redirect_url}" - logger.warning(redirect_message) - print(redirect_message) - - gh_repo_namespace = None - gh_repo_name = None - if redirect_url.startswith("https://github.com/"): - redirect_url_split = redirect_url.split("/") - if len(redirect_url_split) < 3: - return purl_to_cocoapods_org_url_status - gh_repo_namespace = redirect_url_split[-2] - gh_repo_name = redirect_url_split[-1] - - redirect_to_gh_response = utils.get_complete_response(redirect_url) - if "Failed to fetch" in redirect_to_gh_response: - logger.error(redirect_to_gh_response) - print(redirect_to_gh_response) - purl_to_cocoapods_org_url_status["return_message"] = "failed_to_fetch_github_redirect" - return purl_to_cocoapods_org_url_status - elif "not_found" in redirect_to_gh_response: - redirect_to_gh_not_found = f"Redirect to GitHub not found: {redirect_url}" - logger.error(redirect_to_gh_not_found) - print(redirect_to_gh_not_found) - purl_to_cocoapods_org_url_status["return_message"] = "github_redirect_not_found" - return purl_to_cocoapods_org_url_status - - soup = BeautifulSoup(redirect_to_gh_response.text, "html.parser") - head = soup.find("head") - og_url_tag_get_content = None - corrected_name = None - if head: - og_url_tag = head.find("meta", property="og:url") - if og_url_tag: - og_url = og_url_tag.get("content") - og_url_tag_get_content = og_url - corrected_name = og_url_tag_get_content.split('/')[-1] - else: - no_meta_tag = f"'og:url' meta tag not found in redirect_to_gh_response page for {purl}" - print(no_meta_tag) - logger.error(no_meta_tag) - purl_to_cocoapods_org_url_status["return_message"] = "github_redirect_error" - return purl_to_cocoapods_org_url_status - else: - no_head_section = f"\n section not found in redirect_to_gh_response page for {purl}" - print(no_head_section) - logger.error(no_head_section) - purl_to_cocoapods_org_url_status["return_message"] = "github_redirect_error" - return purl_to_cocoapods_org_url_status - - cocoapods_org_version = None - - purl_to_cocoapods_org_url_status["corrected_name"] = corrected_name - purl_to_cocoapods_org_url_status["cocoapods_org_pod_name"] = corrected_name - purl_to_cocoapods_org_url_status["cocoapods_org_gh_repo_owner"] = gh_repo_namespace - purl_to_cocoapods_org_url_status["cocoapods_org_gh_repo_name"] = gh_repo_name - purl_to_cocoapods_org_url_status["cocoapods_org_version"] = cocoapods_org_version - purl_to_cocoapods_org_url_status["return_message"] = "cocoapods_org_redirects_to_github" - return purl_to_cocoapods_org_url_status - else: - purl_to_cocoapods_org_url_status["return_message"] = "cocoapods_org_url_redirects" - return purl_to_cocoapods_org_url_status - - else: - purl_to_cocoapods_org_url_status["return_message"] = None - return purl_to_cocoapods_org_url_status - - -def get_pod_data_with_soup(purl, name, cocoapods_org_url): - purl_to_pod_data_with_soup = {} - cocoapods_org_response = utils.get_complete_response(cocoapods_org_url) - if "Failed to fetch" in cocoapods_org_response: - logger.error(cocoapods_org_response) - print(cocoapods_org_response) - return - - soup = BeautifulSoup(cocoapods_org_response.text, "html.parser") - cocoapods_org_gh_repo_owner = None - cocoapods_org_gh_repo_name = None - cocoapods_org_gh_repo_url = None - cocoapods_org_podspec_url = None - cocoapods_org_pkg_home_url = None - - for sidebar_links in (soup.find_all('ul', class_ = "links" )): - nested_links = sidebar_links.findChildren("a") - for nested_link in nested_links: - link_text = nested_link.text - link_url = nested_link['href'] - if link_text == 'Homepage': - cocoapods_org_pkg_home_url = link_url - elif link_text == 'GitHub Repo': - split_link = link_url.split('/') - cocoapods_org_gh_repo_owner = split_link[-2] - cocoapods_org_gh_repo_name = split_link[-1] - elif link_text == 'See Podspec': - cocoapods_org_podspec_url = link_url - - if cocoapods_org_gh_repo_owner and cocoapods_org_gh_repo_name: - cocoapods_org_gh_repo_url = f"https://github.com/{cocoapods_org_gh_repo_owner}/{cocoapods_org_gh_repo_name}" - cocoapods_org_gh_repo_url_head_request = utils.make_head_request(cocoapods_org_gh_repo_url) - cocoapods_org_gh_repo_url_status_code = cocoapods_org_gh_repo_url_head_request.status_code - purl_to_pod_data_with_soup["cocoapods_org_gh_repo_url_status_code"] = cocoapods_org_gh_repo_url_status_code - - base_path = "https://api.github.com/repos" - api_url = f"{base_path}/{cocoapods_org_gh_repo_owner}/{cocoapods_org_gh_repo_name}" - github_rest_no_exception_response = utils.get_github_rest_no_exception(api_url) - if "Failed to fetch" in github_rest_no_exception_response: - logger.error(f"{github_rest_no_exception_response}") - print(f"{github_rest_no_exception_response}") - - purl_to_pod_data_with_soup["cocoapods_org_gh_repo_owner"] = cocoapods_org_gh_repo_owner - purl_to_pod_data_with_soup["cocoapods_org_gh_repo_name"] = cocoapods_org_gh_repo_name - purl_to_pod_data_with_soup["cocoapods_org_gh_repo_url"] = cocoapods_org_gh_repo_url - purl_to_pod_data_with_soup["cocoapods_org_podspec_url"] = cocoapods_org_podspec_url - purl_to_pod_data_with_soup["cocoapods_org_pkg_home_url"] = cocoapods_org_pkg_home_url - - if cocoapods_org_gh_repo_owner is None or cocoapods_org_gh_repo_name is None: - no_github_repo = f"No GitHub repo found on cocoapods.org for {name}" - print(f"{no_github_repo}") - logger.warning(no_github_repo) - - if cocoapods_org_podspec_url is None: - no_podspec = f"No podspec found on cocoapods.org for {name}" - print(f"{no_podspec}") - logger.warning(no_podspec) - purl_to_pod_data_with_soup["no_podspec"] = no_podspec - - cocoapods_org_version = None - purl_to_pod_data_with_soup["cocoapods_org_version"] = cocoapods_org_version - if cocoapods_org_podspec_url: - cocoapods_org_version = cocoapods_org_podspec_url.split("/")[-2] - - cocoapods_org_pod_name = None - head = soup.find("head") - if head: - og_title_tag = head.find("meta", property="og:title") - if og_title_tag: - og_title = og_title_tag.get("content") - cocoapods_org_pod_name = og_title - else: - no_meta_tag = f"'og:title' meta tag not found in cocoapods.org page for {purl}" - print(no_meta_tag) - logger.error(no_meta_tag) - else: - no_head_section = f"\n section not found in cocoapods.org page for {purl}" - print(no_head_section) - logger.error(no_head_section) - - purl_to_pod_data_with_soup["cocoapods_org_pod_name"] = cocoapods_org_pod_name - input_name = name - if input_name != cocoapods_org_pod_name: - name_change = (f"Input PURL name '{input_name}' analyzed as '{cocoapods_org_pod_name}' per {cocoapods_org_url}") - input_name = cocoapods_org_pod_name - print(f"{name_change}") - logger.warning(name_change) - - return purl_to_pod_data_with_soup - - -def get_cocoapod_tags(spec, cocoapods_org_pod_name): +def get_cocoapod_tags(spec, name): try: response = utils.get_text_response(spec) data = response.strip() for line in data.splitlines(): line = line.strip() - if line.startswith(cocoapods_org_pod_name): + if line.startswith(name): data_list = line.split("/") - if data_list[0] == cocoapods_org_pod_name: + if data_list[0] == name: data_list.pop(0) - sorted_data_list = sorted( - data_list, - key=lambda x: versions.SemverVersion(x), - reverse=True, - ) - return sorted_data_list + return data_list return None except: - print(f"Error retrieving cocoapods tag data from cdn.cocoapods.org") return None @@ -931,11 +745,10 @@ def construct_cocoapods_package( purl, name, hashed_path, - repository_homepage_url, - cocoapods_org_gh_repo_owner, - cocoapods_org_gh_repo_name, - tag, - cocoapods_org_pod_name + cocoapods_org_url, + gh_repo_owner, + gh_repo_name, + tag ): name = name homepage_url = None @@ -947,14 +760,14 @@ def construct_cocoapods_package( declared_license = None primary_language = None - if cocoapods_org_gh_repo_owner and cocoapods_org_gh_repo_name: - name = cocoapods_org_gh_repo_name - namespace = cocoapods_org_gh_repo_owner + if gh_repo_owner and gh_repo_name: base_path = "https://api.github.com/repos" - api_url = f"{base_path}/{namespace}/{name}" - gh_repo_api_response = utils.get_github_rest_no_exception(api_url) + api_url = f"{base_path}/{gh_repo_owner}/{gh_repo_name}" + gh_repo_api_response = utils.get_github_rest(api_url) + gh_repo_api_head_request = utils.make_head_request(api_url) + gh_repo_api_status_code = gh_repo_api_head_request.status_code - if "Failed to fetch" not in gh_repo_api_response: + if gh_repo_api_status_code == 200: homepage_url = gh_repo_api_response.get("homepage") vcs_url = gh_repo_api_response.get("git_url") license_data = gh_repo_api_response.get("license") or {} @@ -962,18 +775,11 @@ def construct_cocoapods_package( primary_language = gh_repo_api_response.get("language") github_url = "https://github.com" - bug_tracking_url = f"{github_url}/{namespace}/{name}/issues" - code_view_url = f"{github_url}/{namespace}/{name}" - - corrected_name = cocoapods_org_pod_name - podspec_api_url = f"https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/{hashed_path}/{corrected_name}/{tag}/{corrected_name}.podspec.json" - podspec_api_response = utils.get_json_response(podspec_api_url) - - if "Failed to fetch" in podspec_api_response: - logger.error(f"{podspec_api_response}") - print(f"{podspec_api_response}") - return + bug_tracking_url = f"{github_url}/{gh_repo_owner}/{gh_repo_name}/issues" + code_view_url = f"{github_url}/{gh_repo_owner}/{gh_repo_name}" + podspec_api_url = f"https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/{hashed_path}/{name}/{tag}/{name}.podspec.json" + podspec_api_response = utils.get_response(podspec_api_url) homepage_url = podspec_api_response.get("homepage") lic = podspec_api_response.get("license") @@ -986,7 +792,6 @@ def construct_cocoapods_package( declared_license = extracted_license_statement source = podspec_api_response.get("source") - vcs_url = None download_url = None if isinstance(source, dict): git_url = source.get("git", "") @@ -994,12 +799,12 @@ def construct_cocoapods_package( if http_url: download_url = http_url if git_url and not http_url: - if git_url.endswith(".git") and "github" in git_url: + if git_url.endswith(".git") and git_url.startswith("https://github.com/"): gh_path = git_url[:-4] - corrected_tag = tag - if source.get("tag") and source.get("tag").startswith("v"): - corrected_tag = source.get("tag") - download_url = f"{gh_path}/archive/refs/tags/{corrected_tag}.tar.gz" + github_tag = source.get("tag") + if github_tag and github_tag.startswith("v"): + tag = github_tag + download_url = f"{gh_path}/archive/refs/tags/{tag}.tar.gz" vcs_url = git_url elif git_url: vcs_url = git_url @@ -1015,9 +820,10 @@ def construct_cocoapods_package( download_url=download_url, declared_license=declared_license, primary_language=primary_language, - repository_homepage_url=repository_homepage_url, + repository_homepage_url=cocoapods_org_url, vcs_url=vcs_url, **purl.to_dict(), ) purl_pkg.version = tag + return purl_pkg diff --git a/src/fetchcode/utils.py b/src/fetchcode/utils.py index b0edfde..e68c434 100644 --- a/src/fetchcode/utils.py +++ b/src/fetchcode/utils.py @@ -170,9 +170,6 @@ def get_github_rest(url): def get_response(url, headers=None): - """ - Generate `Package` object for a `url` string - """ resp = requests.get(url, headers=headers) if resp.status_code == 200: return resp.json() @@ -180,55 +177,20 @@ def get_response(url, headers=None): raise Exception(f"Failed to fetch: {url}") -def get_github_rest_no_exception(url): - headers = None - gh_token = get_github_token() - if gh_token: - headers = { - "Authorization": f"Bearer {gh_token}", - } - - return get_json_response(url, headers) - - -def get_json_response(url, headers=None): - """ - Generate `Package` object for a `url` string - """ - resp = requests.get(url, headers=headers) - if resp.status_code == 200: - return resp.json() - - return f"Failed to fetch: {url}" - - def get_text_response(url, headers=None): resp = requests.get(url, headers=headers) if resp.status_code == 200: return resp.text - return f"Failed to fetch: {url}" - - -def get_complete_response(url, headers=None, params=None): - resp = requests.get(url, headers=headers, params=params) - if resp.status_code == 200: - return resp - elif resp.status_code == 404: - return "not_found" - - return f"Failed to fetch: {url}" + raise Exception(f"Failed to fetch: {url}") def make_head_request(url, headers=None): try: resp = requests.head(url, headers=headers) - return resp - - except requests.exceptions.RequestException as e: - print(f"An error occurred: {e}") - return "cannot_confirm" + except: + raise Exception(f"Failed to fetch: {url}") def get_hashed_path(name): diff --git a/tests/data/cocoapods/afnetworking_github_rest_no_exception_response.json b/tests/data/cocoapods/afnetworking_github_rest_response.json similarity index 99% rename from tests/data/cocoapods/afnetworking_github_rest_no_exception_response.json rename to tests/data/cocoapods/afnetworking_github_rest_response.json index 6ffc8b7..9abf6f4 100644 --- a/tests/data/cocoapods/afnetworking_github_rest_no_exception_response.json +++ b/tests/data/cocoapods/afnetworking_github_rest_response.json @@ -75,7 +75,7 @@ "size": 6129, "stargazers_count": 33331, "watchers_count": 33331, - "language": "Objective-C", + "language": "Objective-c", "has_issues": false, "has_projects": false, "has_downloads": true, diff --git a/tests/data/cocoapods/expected_result_to_dict.json b/tests/data/cocoapods/expected_result_to_dict.json new file mode 100644 index 0000000..a17f7e5 --- /dev/null +++ b/tests/data/cocoapods/expected_result_to_dict.json @@ -0,0 +1,236 @@ +[ + { + "type": "cocoapods", + "namespace": null, + "name": "ASNetworking", + "version": "0.1.5", + "qualifiers": {}, + "subpath": null, + "repository_homepage_url": "https://cocoapods.org/pods/ASNetworking", + "repository_download_url": null, + "api_data_url": null, + "primary_language": "Swift", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/Appspia/ASNetworking", + "download_url": "https://github.com/Appspia/ASNetworking/archive/refs/tags/0.1.5.tar.gz", + "api_url": "https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/5/5/b/ASNetworking/0.1.5/ASNetworking.podspec.json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/Appspia/ASNetworking/issues", + "code_view_url": "https://github.com/Appspia/ASNetworking", + "vcs_url": "https://github.com/Appspia/ASNetworking.git", + "copyright": null, + "license_expression": null, + "declared_license": { + "type": "MIT", + "file": "LICENSE" + }, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:cocoapods/ASNetworking@0.1.5" + }, + { + "type": "cocoapods", + "namespace": null, + "name": "ASNetworking", + "version": "0.1.4", + "qualifiers": {}, + "subpath": null, + "repository_homepage_url": "https://cocoapods.org/pods/ASNetworking", + "repository_download_url": null, + "api_data_url": null, + "primary_language": "Swift", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/Appspia/ASNetworking", + "download_url": "https://github.com/Appspia/ASNetworking/archive/refs/tags/0.1.4.tar.gz", + "api_url": "https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/5/5/b/ASNetworking/0.1.4/ASNetworking.podspec.json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/Appspia/ASNetworking/issues", + "code_view_url": "https://github.com/Appspia/ASNetworking", + "vcs_url": "https://github.com/Appspia/ASNetworking.git", + "copyright": null, + "license_expression": null, + "declared_license": { + "type": "MIT", + "file": "LICENSE" + }, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:cocoapods/ASNetworking@0.1.4" + }, + { + "type": "cocoapods", + "namespace": null, + "name": "ASNetworking", + "version": "0.1.3", + "qualifiers": {}, + "subpath": null, + "repository_homepage_url": "https://cocoapods.org/pods/ASNetworking", + "repository_download_url": null, + "api_data_url": null, + "primary_language": "Swift", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/Appspia/ASNetworking", + "download_url": "https://github.com/Appspia/ASNetworking/archive/refs/tags/0.1.3.tar.gz", + "api_url": "https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/5/5/b/ASNetworking/0.1.3/ASNetworking.podspec.json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/Appspia/ASNetworking/issues", + "code_view_url": "https://github.com/Appspia/ASNetworking", + "vcs_url": "https://github.com/Appspia/ASNetworking.git", + "copyright": null, + "license_expression": null, + "declared_license": { + "type": "MIT", + "file": "LICENSE" + }, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:cocoapods/ASNetworking@0.1.3" + }, + { + "type": "cocoapods", + "namespace": null, + "name": "ASNetworking", + "version": "0.1.2", + "qualifiers": {}, + "subpath": null, + "repository_homepage_url": "https://cocoapods.org/pods/ASNetworking", + "repository_download_url": null, + "api_data_url": null, + "primary_language": "Swift", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/Appspia/ASNetworking", + "download_url": "https://github.com/Appspia/ASNetworking/archive/refs/tags/0.1.2.tar.gz", + "api_url": "https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/5/5/b/ASNetworking/0.1.2/ASNetworking.podspec.json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/Appspia/ASNetworking/issues", + "code_view_url": "https://github.com/Appspia/ASNetworking", + "vcs_url": "https://github.com/Appspia/ASNetworking.git", + "copyright": null, + "license_expression": null, + "declared_license": { + "type": "MIT", + "file": "LICENSE" + }, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:cocoapods/ASNetworking@0.1.2" + }, + { + "type": "cocoapods", + "namespace": null, + "name": "ASNetworking", + "version": "0.1.1", + "qualifiers": {}, + "subpath": null, + "repository_homepage_url": "https://cocoapods.org/pods/ASNetworking", + "repository_download_url": null, + "api_data_url": null, + "primary_language": "Swift", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/Appspia/ASNetworking", + "download_url": "https://github.com/Appspia/ASNetworking/archive/refs/tags/0.1.1.tar.gz", + "api_url": "https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/5/5/b/ASNetworking/0.1.1/ASNetworking.podspec.json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/Appspia/ASNetworking/issues", + "code_view_url": "https://github.com/Appspia/ASNetworking", + "vcs_url": "https://github.com/Appspia/ASNetworking.git", + "copyright": null, + "license_expression": null, + "declared_license": { + "type": "MIT", + "file": "LICENSE" + }, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:cocoapods/ASNetworking@0.1.1" + }, + { + "type": "cocoapods", + "namespace": null, + "name": "ASNetworking", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "repository_homepage_url": "https://cocoapods.org/pods/ASNetworking", + "repository_download_url": null, + "api_data_url": null, + "primary_language": "Swift", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/Appspia/ASNetworking", + "download_url": "https://github.com/Appspia/ASNetworking/archive/refs/tags/0.1.0.tar.gz", + "api_url": "https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/5/5/b/ASNetworking/0.1.0/ASNetworking.podspec.json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/Appspia/ASNetworking/issues", + "code_view_url": "https://github.com/Appspia/ASNetworking", + "vcs_url": "https://github.com/Appspia/ASNetworking.git", + "copyright": null, + "license_expression": null, + "declared_license": { + "type": "MIT", + "file": "LICENSE" + }, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:cocoapods/ASNetworking@0.1.0" + } +] diff --git a/tests/data/cocoapods/get_json_response_kvllibraries.json b/tests/data/cocoapods/get_response_kvllibraries.json similarity index 100% rename from tests/data/cocoapods/get_json_response_kvllibraries.json rename to tests/data/cocoapods/get_response_kvllibraries.json diff --git a/tests/data/cocoapods/mock_get_github_rest_return_value.json b/tests/data/cocoapods/mock_get_github_rest_return_value.json new file mode 100644 index 0000000..75e7dde --- /dev/null +++ b/tests/data/cocoapods/mock_get_github_rest_return_value.json @@ -0,0 +1,110 @@ +{ + "id": 191114033, + "node_id": "MDEwOlJlcG9zaXRvcnkxOTExMTQwMzM=", + "name": "ASNetworking", + "full_name": "Appspia/ASNetworking", + "private": false, + "owner": { + "login": "Appspia", + "id": 26921895, + "node_id": "MDQ6VXNlcjI2OTIxODk1", + "avatar_url": "https://avatars.githubusercontent.com/u/26921895?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Appspia", + "html_url": "https://github.com/Appspia", + "followers_url": "https://api.github.com/users/Appspia/followers", + "following_url": "https://api.github.com/users/Appspia/following{/other_user}", + "gists_url": "https://api.github.com/users/Appspia/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Appspia/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Appspia/subscriptions", + "organizations_url": "https://api.github.com/users/Appspia/orgs", + "repos_url": "https://api.github.com/users/Appspia/repos", + "events_url": "https://api.github.com/users/Appspia/events{/privacy}", + "received_events_url": "https://api.github.com/users/Appspia/received_events", + "type": "User", + "site_admin": false + }, + "html_url": "https://github.com/Appspia/ASNetworking", + "description": null, + "fork": false, + "url": "https://api.github.com/repos/Appspia/ASNetworking", + "forks_url": "https://api.github.com/repos/Appspia/ASNetworking/forks", + "keys_url": "https://api.github.com/repos/Appspia/ASNetworking/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/Appspia/ASNetworking/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/Appspia/ASNetworking/teams", + "hooks_url": "https://api.github.com/repos/Appspia/ASNetworking/hooks", + "issue_events_url": "https://api.github.com/repos/Appspia/ASNetworking/issues/events{/number}", + "events_url": "https://api.github.com/repos/Appspia/ASNetworking/events", + "assignees_url": "https://api.github.com/repos/Appspia/ASNetworking/assignees{/user}", + "branches_url": "https://api.github.com/repos/Appspia/ASNetworking/branches{/branch}", + "tags_url": "https://api.github.com/repos/Appspia/ASNetworking/tags", + "blobs_url": "https://api.github.com/repos/Appspia/ASNetworking/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/Appspia/ASNetworking/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/Appspia/ASNetworking/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/Appspia/ASNetworking/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/Appspia/ASNetworking/statuses/{sha}", + "languages_url": "https://api.github.com/repos/Appspia/ASNetworking/languages", + "stargazers_url": "https://api.github.com/repos/Appspia/ASNetworking/stargazers", + "contributors_url": "https://api.github.com/repos/Appspia/ASNetworking/contributors", + "subscribers_url": "https://api.github.com/repos/Appspia/ASNetworking/subscribers", + "subscription_url": "https://api.github.com/repos/Appspia/ASNetworking/subscription", + "commits_url": "https://api.github.com/repos/Appspia/ASNetworking/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/Appspia/ASNetworking/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/Appspia/ASNetworking/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/Appspia/ASNetworking/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/Appspia/ASNetworking/contents/{+path}", + "compare_url": "https://api.github.com/repos/Appspia/ASNetworking/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/Appspia/ASNetworking/merges", + "archive_url": "https://api.github.com/repos/Appspia/ASNetworking/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/Appspia/ASNetworking/downloads", + "issues_url": "https://api.github.com/repos/Appspia/ASNetworking/issues{/number}", + "pulls_url": "https://api.github.com/repos/Appspia/ASNetworking/pulls{/number}", + "milestones_url": "https://api.github.com/repos/Appspia/ASNetworking/milestones{/number}", + "notifications_url": "https://api.github.com/repos/Appspia/ASNetworking/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/Appspia/ASNetworking/labels{/name}", + "releases_url": "https://api.github.com/repos/Appspia/ASNetworking/releases{/id}", + "deployments_url": "https://api.github.com/repos/Appspia/ASNetworking/deployments", + "created_at": "2019-06-10T06:55:56Z", + "updated_at": "2022-07-01T07:10:28Z", + "pushed_at": "2022-11-10T09:56:57Z", + "git_url": "git://github.com/Appspia/ASNetworking.git", + "ssh_url": "git@github.com:Appspia/ASNetworking.git", + "clone_url": "https://github.com/Appspia/ASNetworking.git", + "svn_url": "https://github.com/Appspia/ASNetworking", + "homepage": null, + "size": 53, + "stargazers_count": 0, + "watchers_count": 0, + "language": "Swift", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "has_discussions": false, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 0, + "license": null, + "allow_forking": true, + "is_template": false, + "web_commit_signoff_required": false, + "topics": [], + "visibility": "public", + "forks": 0, + "open_issues": 0, + "watchers": 0, + "default_branch": "master", + "permissions": { + "admin": false, + "maintain": false, + "push": false, + "triage": false, + "pull": true + }, + "temp_clone_token": "", + "network_count": 0, + "subscribers_count": 0 +} diff --git a/tests/data/cocoapods/mock_get_response_side_effect.json b/tests/data/cocoapods/mock_get_response_side_effect.json new file mode 100644 index 0000000..2b88533 --- /dev/null +++ b/tests/data/cocoapods/mock_get_response_side_effect.json @@ -0,0 +1,139 @@ +[ + { + "name": "ASNetworking", + "version": "0.1.5", + "summary": "Simple HTTP Networking in Swift", + "homepage": "https://github.com/Appspia/ASNetworking", + "license": { + "type": "MIT", + "file": "LICENSE" + }, + "authors": { + "Appspia": "appspia@gmail.com" + }, + "source": { + "git": "https://github.com/Appspia/ASNetworking.git", + "tag": "0.1.5" + }, + "platforms": { + "ios": "9.0" + }, + "swift_versions": "5.0", + "source_files": "ASNetworking/**/*" + }, + { + "name": "ASNetworking", + "version": "0.1.4", + "summary": "Simple HTTP Networking in Swift", + "description": "TODO: Add long description of the pod here.", + "homepage": "https://github.com/Appspia/ASNetworking", + "license": { + "type": "MIT", + "file": "LICENSE" + }, + "authors": { + "Appspia": "appspia@gmail.com" + }, + "source": { + "git": "https://github.com/Appspia/ASNetworking.git", + "tag": "0.1.4" + }, + "platforms": { + "ios": "8.0" + }, + "swift_version": "4.2", + "source_files": "ASNetworking/**/*" + }, + { + "name": "ASNetworking", + "version": "0.1.3", + "summary": "Simple HTTP Networking in Swift", + "description": "TODO: Add long description of the pod here.", + "homepage": "https://github.com/Appspia/ASNetworking", + "license": { + "type": "MIT", + "file": "LICENSE" + }, + "authors": { + "Appspia": "appspia@gmail.com" + }, + "source": { + "git": "https://github.com/Appspia/ASNetworking.git", + "tag": "0.1.3" + }, + "platforms": { + "ios": "8.0" + }, + "swift_version": "4.2", + "source_files": "ASNetworking/**/*" + }, + { + "name": "ASNetworking", + "version": "0.1.2", + "summary": "Simple HTTP Networking in Swift", + "description": "TODO: Add long description of the pod here.", + "homepage": "https://github.com/Appspia/ASNetworking", + "license": { + "type": "MIT", + "file": "LICENSE" + }, + "authors": { + "Appspia": "appspia@gmail.com" + }, + "source": { + "git": "https://github.com/Appspia/ASNetworking.git", + "tag": "0.1.2" + }, + "platforms": { + "ios": "8.0" + }, + "swift_version": "4.2", + "source_files": "ASNetworking/**/*" + }, + { + "name": "ASNetworking", + "version": "0.1.1", + "summary": "Simple HTTP Networking in Swift", + "description": "TODO: Add long description of the pod here.", + "homepage": "https://github.com/Appspia/ASNetworking", + "license": { + "type": "MIT", + "file": "LICENSE" + }, + "authors": { + "Appspia": "appspia@gmail.com" + }, + "source": { + "git": "https://github.com/Appspia/ASNetworking.git", + "tag": "0.1.1" + }, + "platforms": { + "ios": "8.0" + }, + "swift_version": "4.2", + "source_files": "ASNetworking/**/*" + }, + { + "name": "ASNetworking", + "version": "0.1.0", + "summary": "Simple HTTP Networking in Swift", + "description": "TODO: Add long description of the pod here.", + "homepage": "https://github.com/Appspia/ASNetworking", + "license": { + "type": "MIT", + "file": "LICENSE" + }, + "authors": { + "Appspia": "appspia@gmail.com" + }, + "source": { + "git": "https://github.com/Appspia/ASNetworking.git", + "tag": "0.1.0" + }, + "platforms": { + "ios": "8.0" + }, + "swift_version": "4.2", + "source_files": "ASNetworking/**/*" + } +] diff --git a/tests/test_package.py b/tests/test_package.py index e4ea94d..6482ab1 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -15,7 +15,6 @@ # specific language governing permissions and limitations under the License. import json -from collections import OrderedDict from unittest import TestCase from unittest import mock @@ -26,9 +25,6 @@ from fetchcode.package import info from fetchcode.package_util import construct_cocoapods_package from fetchcode.package_util import get_cocoapod_tags -from fetchcode.package_util import get_cocoapods_org_url_status -from fetchcode.package_util import get_pod_data_with_soup -from fetchcode.packagedcode_models import Package def file_data(file_name): @@ -37,12 +33,6 @@ def file_data(file_name): return json.loads(data) -def file_data_text(file_name): - with open(file_name) as file: - data = file.read() - return data - - def match_data(packages, expected_data): data = [dict(p.to_dict()) for p in packages] expected_data_dict = dict(expected_data) @@ -117,152 +107,40 @@ def test_tuby_package_with_invalid_url(mock_get): assert "Failed to fetch: https://rubygems.org/api/v1/gems/file.json" == e_info -# 2024-05-07 Tuesday 18:08:04. Work-in-progress. The output data leads me to believe there's still at least one live call out to the Internet. -# @mock.patch("fetchcode.package_util.construct_cocoapods_package") # variable containing result is `tag_pkg` -# @mock.patch("fetchcode.package_util.get_cocoapod_tags") # variable containing result is `data_list` -# @mock.patch("fetchcode.utils.get_hashed_path") -# @mock.patch("fetchcode.package_util.get_pod_data_with_soup") -# @mock.patch("fetchcode.package_util.get_cocoapods_org_url_status") -# def test_get_cocoapods_data_from_purl(mock_get_cocoapods_org_url_status, mock_get_pod_data_with_soup, mock_get_hashed_path, mock_get_cocoapod_tags, mock_construct_cocoapods_package): -# # def test_get_cocoapods_data_from_purl(): -# # print(f"\ntest construction in progress....") -# mock_get_cocoapods_org_url_status.return_value = {'return_message': None} - -# mock_get_pod_data_with_soup.return_value = { -# 'cocoapods_org_gh_repo_url_status_code': 200, -# 'cocoapods_org_gh_repo_owner': 'Appspia', -# 'cocoapods_org_gh_repo_name': 'ASNetworking', -# 'cocoapods_org_gh_repo_url': 'https://github.com/Appspia/ASNetworking', -# 'cocoapods_org_podspec_url': 'https://github.com/CocoaPods/Specs/blob/master/Specs/5/5/b/ASNetworking/0.1.5/ASNetworking.podspec.json', -# 'cocoapods_org_pkg_home_url': None, -# 'cocoapods_org_version': None, -# 'cocoapods_org_pod_name': 'ASNetworking', -# } - -# mock_get_hashed_path.return_value = "5/5/b" - -# mock_get_cocoapod_tags.side_effect = [ -# '0.1.5', -# '0.1.4', -# '0.1.3', -# '0.1.2', -# '0.1.1', -# '0.1.0', -# ] - -# mock_construct_cocoapods_package.side_effect = [ -# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.5'), -# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.4'), -# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.3'), -# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.2'), -# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.1'), -# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.0'), -# ] - -# expected_result = [ -# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.5'), -# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.4'), -# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.3'), -# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.2'), -# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.1'), -# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.0'), -# ] - -# purl = "pkg:cocoapods/ASNetworking" - -# actual_result = get_cocoapods_data_from_purl(purl) -# print(f"\nactual_result = {actual_result}") - -# # for pkg in actual_result: -# # print(pkg.to_dict()) - -# # assert list(actual_result) == expected_result - -# for pkg, expected_pkg in zip(list(actual_result), expected_result): -# assert pkg.to_dict() == expected_pkg.to_dict() - - - - - -@mock.patch("fetchcode.utils.make_head_request") -def test_get_cocoapods_org_url_status(mock_make_head_request): - mock_response = mock.Mock() - mock_response.status_code = 302 - mock_response.text = "The cocoapods.org URL https://cocoapods.org/pods/BSSimpleHTTPNetworking redirects to https://github.com/juxingzhutou/BSSimpleHTTPNetworking" - mock_response.headers = { - 'Date': 'Thu, 02 May 2024 06:02:10 GMT', - 'Content-Type': 'text/html;charset=utf-8', - 'Connection': 'keep-alive', - 'Report-To': '{"group":"heroku-nel","max_age":3600,"endpoints":[{"url":"https://nel.heroku.com/reports?ts=1714629728&sid=c46efe9b-d3d2-4a0c-8c76-bfafa16c5add&s=rPI0KHQY0J7GvkjgHpmcuMxWDuTga0k8UEFRezWRyrU%3D"}]}', - 'Reporting-Endpoints': 'heroku-nel=https://nel.heroku.com/reports?ts=1714629728&sid=c46efe9b-d3d2-4a0c-8c76-bfafa16c5add&s=rPI0KHQY0J7GvkjgHpmcuMxWDuTga0k8UEFRezWRyrU%3D', - 'Nel': '{"report_to":"heroku-nel","max_age":3600,"success_fraction":0.005,"failure_fraction":0.05,"response_headers":["Via"]}', - 'Cache-Control': 'public, max-age=20, s-maxage=60', - 'Location': 'https://github.com/juxingzhutou/BSSimpleHTTPNetworking', - 'X-Xss-Protection': '1; mode=block', - 'X-Content-Type-Options': 'nosniff', - 'X-Frame-Options': 'SAMEORIGIN', - 'Via': '1.1 vegur', - 'CF-Cache-Status': 'HIT', - 'Age': '2', - 'Vary': 'Accept-Encoding', - 'Server': 'cloudflare', - 'CF-RAY': '87d5cd05bd2cf973-SJC', - } - mock_make_head_request.return_value = mock_response - - purl = "pkg:cocoapods/BSSimpleHTTPNetworking" - name = "BSSimpleHTTPNetworking" - cocoapods_org_url = "https://cocoapods.org/pods/BSSimpleHTTPNetworking" - response = get_cocoapods_org_url_status(purl, name, cocoapods_org_url) - - assert response == { - 'corrected_name': 'BSSimpleHTTPNetworking', - 'cocoapods_org_pod_name': 'BSSimpleHTTPNetworking', - 'cocoapods_org_gh_repo_owner': 'juxingzhutou', - 'cocoapods_org_gh_repo_name': 'BSSimpleHTTPNetworking', - 'cocoapods_org_version': None, - 'return_message': 'cocoapods_org_redirects_to_github', - } - - -@mock.patch("fetchcode.utils.get_github_rest_no_exception") -@mock.patch("fetchcode.utils.make_head_request") -@mock.patch("fetchcode.utils.get_complete_response") -def test_get_pod_data_with_soup(mock_get_complete_response, mock_make_head_request, mock_get_github_rest_no_exception): - mock_complete_response = mock.MagicMock() - mock_complete_response.status_code = 200 - mock_complete_response.text = file_data_text("tests/data/cocoapods/afnetworking_response_text.txt") - mock_get_complete_response.side_effect = [mock_complete_response] - - mock_head_request_response = mock.MagicMock() - mock_head_request_response.status_code = 200 - mock_make_head_request.side_effect = [mock_head_request_response] - - mock_get_github_rest_no_exception.side_effect = [file_data("tests/data/cocoapods/afnetworking_github_rest_no_exception_response.json")] - - purl = PackageURL.from_string("pkg:cocoapods/AFNetworking@4.0.1") - name = "AFNetworking" - cocoapods_org_url = "https://cocoapods.org/pods/AFNetworking" - - soup_data = get_pod_data_with_soup(purl, name, cocoapods_org_url) - expected = { - 'cocoapods_org_gh_repo_url_status_code': 200, - 'cocoapods_org_gh_repo_owner': 'AFNetworking', - 'cocoapods_org_gh_repo_name': 'AFNetworking', - 'cocoapods_org_gh_repo_url': 'https://github.com/AFNetworking/AFNetworking', - 'cocoapods_org_podspec_url': 'https://github.com/CocoaPods/Specs/blob/master/Specs/a/7/5/AFNetworking/4.0.1/AFNetworking.podspec.json', - 'cocoapods_org_pkg_home_url': None, - 'cocoapods_org_version': None, - 'cocoapods_org_pod_name': 'AFNetworking', - } - - assert soup_data == expected - - -@mock.patch("fetchcode.utils.get_text_response") +@mock.patch("fetchcode.package_util.utils.get_response") +@mock.patch("fetchcode.package_util.utils.get_github_rest") +@mock.patch("fetchcode.package.get_cocoapod_tags") +@mock.patch("fetchcode.package.get_hashed_path") +def test_get_cocoapods_data_from_purl( + mock_get_hashed_path, + mock_get_cocoapod_tags, + mock_get_github_rest, + mock_get_response, +): + mock_get_hashed_path.return_value = "5/5/b" + mock_get_cocoapod_tags.return_value = [ + '0.1.5', + '0.1.4', + '0.1.3', + '0.1.2', + '0.1.1', + '0.1.0', + ] + mock_get_github_rest.return_value = file_data("tests/data/cocoapods/mock_get_github_rest_return_value.json") + mock_get_response.side_effect = file_json("tests/data/cocoapods/mock_get_response_side_effect.json") + expected_result_to_dict = file_json("tests/data/cocoapods/expected_result_to_dict.json") + purl = "pkg:cocoapods/ASNetworking" + actual_result = get_cocoapods_data_from_purl(purl) + + for pkg, expected_pkg_to_dict in zip(list(actual_result), expected_result_to_dict): + pkg_to_json = json.dumps(pkg.to_dict()) + expected_pkg_to_dict_json_dumps = json.dumps(expected_pkg_to_dict) + assert pkg_to_json == expected_pkg_to_dict_json_dumps + + +@mock.patch("fetchcode.package_util.utils.get_text_response") def test_get_cocoapod_tags(mock_get): - side_effect = [file_data_text("tests/data/cocoapods/cocoapod_all_pods_versions_5_1_f.txt")] + side_effect = [file_content("tests/data/cocoapods/cocoapod_all_pods_versions_5_1_f.txt")] mock_get.side_effect = side_effect cocoapods_org_pod_name = "DeptFlow" api = "https://cdn.cocoapods.org" @@ -272,28 +150,25 @@ def test_get_cocoapod_tags(mock_get): spec = f"{api}/{file_prefix}{hashed_path_underscore}.txt" expected_data = ['0.3.0', '0.2.0', '0.1.1', '0.1.0'] data_list = get_cocoapod_tags(spec, cocoapods_org_pod_name) - match_data_list(data_list, expected_data) -@mock.patch("fetchcode.utils.get_json_response") -@mock.patch("fetchcode.utils.get_github_rest_no_exception") -def test_construct_cocoapods_package(mock_get_github_rest_no_exception, mock_get_json_response): - mock_get_github_rest_no_exception.return_value = "Failed to fetch: https://api.github.com/repos/KevalPatel94/KVLLibraries" - mock_get_json_response.return_value = file_data("tests/data/cocoapods/get_json_response_kvllibraries.json") - +@mock.patch("fetchcode.package_util.utils.get_response") +@mock.patch("fetchcode.package_util.utils.get_github_rest") +def test_construct_cocoapods_package(mock_get_github_rest, mock_get_response): + mock_get_github_rest.return_value = "Failed to fetch: https://api.github.com/repos/KevalPatel94/KVLLibraries" + mock_get_response.return_value = file_data("tests/data/cocoapods/get_response_kvllibraries.json") expected_construct_cocoapods_package = file_data("tests/data/cocoapods/expected_construct_cocoapods_package.json") purl = PackageURL.from_string("pkg:cocoapods/KVLLibraries") name = "KVLLibraries" hashed_path = "5/1/f" repository_homepage_url = "https://cocoapods.org/pods/KVLLibraries" - cocoapods_org_gh_repo_owner = "KevalPatel94" - cocoapods_org_gh_repo_name = "KVLLibraries" - cocoapods_org_pod_name = "KVLLibraries" + gh_repo_owner = "KevalPatel94" + gh_repo_name = "KVLLibraries" tag = "1.1.0" - actual_output = construct_cocoapods_package(purl, name, hashed_path, repository_homepage_url, cocoapods_org_gh_repo_owner, cocoapods_org_gh_repo_name, tag, cocoapods_org_pod_name) + actual_output = construct_cocoapods_package(purl, name, hashed_path, repository_homepage_url, gh_repo_owner, gh_repo_name, tag) actual = json.dumps(actual_output.to_dict()) expected = json.dumps(expected_construct_cocoapods_package) assert actual == expected