From 801e740b7f09ff2bacb3e991c438d31d92498ec5 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Tue, 7 May 2024 18:20:55 -0700 Subject: [PATCH] Refactor code including dictionary access and tests #116 Reference: https://github.com/nexB/fetchcode/issues/116 Signed-off-by: John M. Horan --- src/fetchcode/package.py | 333 ++------ src/fetchcode/package_util.py | 298 ++++++++ src/fetchcode/utils.py | 14 +- tests/data/cocoapods/AFNetworking.html | 721 ++++++++++++++++++ ...ing_github_rest_no_exception_response.json | 137 ++++ .../cocoapods/afnetworking_response_text.txt | 445 +++++++++++ .../cocoapod_all_pods_versions_5_1_f.txt | 18 + .../expected_construct_cocoapods_package.json | 36 + .../get_json_response_kvllibraries.json | 19 + .../cocoapods/pod_summary_kvllibraries.json | 17 + tests/test_package.py | 202 +++++ 11 files changed, 1982 insertions(+), 258 deletions(-) create mode 100644 tests/data/cocoapods/AFNetworking.html create mode 100644 tests/data/cocoapods/afnetworking_github_rest_no_exception_response.json create mode 100644 tests/data/cocoapods/afnetworking_response_text.txt create mode 100644 tests/data/cocoapods/cocoapod_all_pods_versions_5_1_f.txt create mode 100644 tests/data/cocoapods/expected_construct_cocoapods_package.json create mode 100644 tests/data/cocoapods/get_json_response_kvllibraries.json create mode 100644 tests/data/cocoapods/pod_summary_kvllibraries.json diff --git a/src/fetchcode/package.py b/src/fetchcode/package.py index 2d61aeac..b9035c80 100644 --- a/src/fetchcode/package.py +++ b/src/fetchcode/package.py @@ -15,7 +15,6 @@ # specific language governing permissions and limitations under the License. import dataclasses -import json import logging import os import re @@ -24,11 +23,9 @@ from urllib.parse import urljoin import htmllistparse -from bs4 import BeautifulSoup from packageurl import PackageURL from packageurl.contrib.route import NoRouteAvailable from packageurl.contrib.route import Router -from univers import versions from fetchcode.package_util import GITHUB_SOURCE_BY_PACKAGE from fetchcode.package_util import IPKG_RELEASES @@ -37,14 +34,13 @@ from fetchcode.package_util import GitHubSource from fetchcode.package_util import MiniupnpPackagesGitHubSource from fetchcode.package_util import OpenSSLGitHubSource +from fetchcode.package_util import construct_cocoapods_package +from fetchcode.package_util import get_cocoapod_tags +from fetchcode.package_util import get_cocoapods_org_url_status +from fetchcode.package_util import get_pod_data_with_soup from fetchcode.packagedcode_models import Package -from fetchcode.utils import get_complete_response -from fetchcode.utils import get_github_rest -from fetchcode.utils import get_github_rest_no_exception from fetchcode.utils import get_hashed_path -from fetchcode.utils import get_json_response from fetchcode.utils import get_response -from fetchcode.utils import make_head_request router = Router() @@ -378,139 +374,91 @@ def get_gnu_data_from_purl(purl): @router.route("pkg:cocoapods/.*") def get_cocoapods_data_from_purl(purl): + """ + Generate `Package` object from the `purl` string of cocoapods type + """ logging.basicConfig( filename=LOG_FILE_LOCATION, level=logging.WARN, format="%(levelname)s - %(message)s", filemode="w", ) - input_purl = purl + purl = PackageURL.from_string(purl) name = purl.name version = purl.version cocoapods_org_url = f"https://cocoapods.org/pods/{name}" repository_homepage_url = f"https://cocoapods.org/pods/{name}" - # This dictionary helped me monitor the values as I worked on the code. - # Only 2 key-value pairs are currently used below to define variables: - # `input_name` and `cocoapods_org_pod_name`. - - pod_summary = {} - pod_summary["input_purl"] = input_purl - pod_summary["input_name"] = name - pod_summary["cocoapods_org_url"] = cocoapods_org_url - pod_summary["repository_homepage_url"] = repository_homepage_url - pod_summary["no_github_repo"] = None - pod_summary["gh_repo_four_o_four"] = None - pod_summary["http_url"] = None - - cocoapods_org_url_head_request = make_head_request(cocoapods_org_url) - cocoapods_org_url_status_code = cocoapods_org_url_head_request.status_code - pod_summary["cocoapods_org_url_status_code"] = cocoapods_org_url_status_code - if cocoapods_org_url_status_code == 404: - logger.error(f"cocoapods_org_url not found for {name}") - return - elif cocoapods_org_url_status_code == 302: - redirect_url = cocoapods_org_url_head_request.headers['Location'] - redirect_message = f"The cocoapods.org URL {cocoapods_org_url} redirects to {redirect_url}" - logger.error(redirect_message) - print(redirect_message) - return + purl_to_cocoapods_org_url_status = get_cocoapods_org_url_status(purl, name, cocoapods_org_url) + cocoa_org_url_status = purl_to_cocoapods_org_url_status["return_message"] - cocoapods_org_response = get_complete_response(cocoapods_org_url) - if "Failed to fetch" in cocoapods_org_response: - logger.error(cocoapods_org_response) - print(cocoapods_org_response) + status_values = [ + "cocoapods_org_redirects_to_github", + "cocoapods_org_url_redirects", + "failed_to_fetch_github_redirect", + "github_redirect_error", + "github_redirect_not_found", + ] + + cocoa_org_url_status_code = None + if cocoa_org_url_status == "cocoapods_org_url_not_found": + cocoa_org_url_status_code = 404 + elif cocoa_org_url_status == "cocoapods_org_url_temporarily_unavailable": + cocoa_org_url_status_code = 503 + elif any(cocoa_org_url_status == status for status in status_values): + cocoa_org_url_status_code = 302 + + if ( + cocoa_org_url_status == "cocoapods_org_url_not_found" + or cocoa_org_url_status == "cocoapods_org_url_redirects" + or cocoa_org_url_status == "cocoapods_org_url_temporarily_unavailable" + or cocoa_org_url_status == "failed_to_fetch_github_redirect" + or cocoa_org_url_status == "github_redirect_error" + or cocoa_org_url_status == "github_redirect_not_found" + ): return - soup = BeautifulSoup(cocoapods_org_response.text, "html.parser") + purl_to_pod_data_with_soup = {} + if cocoa_org_url_status_code != 302 and cocoa_org_url_status_code != 503: + purl_to_pod_data_with_soup = get_pod_data_with_soup(purl, name, cocoapods_org_url) - cocoapods_org_gh_repo_owner = None - cocoapods_org_gh_repo_name = None - cocoapods_org_gh_repo_url = None - cocoapods_org_podspec_url = None - cocoapods_org_pkg_home_url = None - - for sidebar_links in (soup.find_all('ul', class_ = "links" )): - nested_links = sidebar_links.findChildren("a") - for nested_link in nested_links: - link_text = nested_link.text - link_url = nested_link['href'] - if link_text == 'Homepage': - cocoapods_org_pkg_home_url = link_url - elif link_text == 'GitHub Repo': - split_link = link_url.split('/') - cocoapods_org_gh_repo_owner = split_link[-2] - cocoapods_org_gh_repo_name = split_link[-1] - elif link_text == 'See Podspec': - cocoapods_org_podspec_url = link_url - - if cocoapods_org_gh_repo_owner and cocoapods_org_gh_repo_name: - cocoapods_org_gh_repo_url = f"https://github.com/{cocoapods_org_gh_repo_owner}/{cocoapods_org_gh_repo_name}" - cocoapods_org_gh_repo_url_head_request = make_head_request(cocoapods_org_gh_repo_url) - cocoapods_org_gh_repo_url_status_code = cocoapods_org_gh_repo_url_head_request.status_code - pod_summary["cocoapods_org_gh_repo_url_status_code"] = cocoapods_org_gh_repo_url_status_code - if cocoapods_org_gh_repo_url_status_code == 404: - gh_repo_four_o_four = f"The cocoapods.org GitHub repo url for {name} returns 404" - logger.error(gh_repo_four_o_four) - print(gh_repo_four_o_four) - pod_summary["gh_repo_four_o_four"] = gh_repo_four_o_four - - name = cocoapods_org_gh_repo_name - base_path = "https://api.github.com/repos" - api_url = f"{base_path}/{cocoapods_org_gh_repo_owner}/{cocoapods_org_gh_repo_name}" - response = get_github_rest_no_exception(api_url) - - if "Failed to fetch" in response: - logger.error(f"{response}") - print(f"{response}") - - pod_summary["cocoapods_org_gh_repo_owner"] = cocoapods_org_gh_repo_owner - pod_summary["cocoapods_org_gh_repo_name"] = cocoapods_org_gh_repo_name - pod_summary["cocoapods_org_gh_repo_url"] = cocoapods_org_gh_repo_url - pod_summary["cocoapods_org_podspec_url"] = cocoapods_org_podspec_url - pod_summary["cocoapods_org_pkg_home_url"] = cocoapods_org_pkg_home_url - - if cocoapods_org_gh_repo_owner is None or cocoapods_org_gh_repo_name is None: - no_github_repo = f"No GitHub repo found on cocoapods.org for {name}" - print(f"{no_github_repo}") - logger.warning(no_github_repo) - pod_summary["no_github_repo"] = no_github_repo - - if cocoapods_org_podspec_url is None: - no_podspec = f"No podspec found on cocoapods.org for {name}" - print(f"{no_podspec}") - logger.warning(no_podspec) - pod_summary["no_podspec"] = no_podspec + cocoapods_org_pod_name = None + if purl_to_pod_data_with_soup.get('cocoapods_org_pod_name') is not None: + cocoapods_org_pod_name = purl_to_pod_data_with_soup["cocoapods_org_pod_name"] + elif purl_to_cocoapods_org_url_status.get('cocoapods_org_pod_name') is not None: + cocoapods_org_pod_name = purl_to_cocoapods_org_url_status["cocoapods_org_pod_name"] cocoapods_org_version = None - if cocoapods_org_podspec_url: - cocoapods_org_version = cocoapods_org_podspec_url.split("/")[-2] + cocoapods_org_gh_repo_owner = None + cocoapods_org_gh_repo_name = None - cocoapods_org_pod_name = None - head = soup.find("head") - if head: - og_title_tag = head.find("meta", property="og:title") - if og_title_tag: - og_title = og_title_tag.get("content") - cocoapods_org_pod_name = og_title - else: - no_meta_tag = f"'og:title' meta tag not found in cocoapods.org page for {purl}" - print(no_meta_tag) - logger.error(no_meta_tag) - else: - no_head_section = f"\n section not found in cocoapods.org page for {purl}" - print(no_head_section) - logger.error(no_head_section) - - pod_summary["cocoapods_org_pod_name"] = cocoapods_org_pod_name - - input_name = pod_summary["input_name"] - if input_name != cocoapods_org_pod_name: - name_change = (f"Input PURL name '{input_name}' analyzed as '{cocoapods_org_pod_name}' per {cocoapods_org_url}") - input_name = cocoapods_org_pod_name - print(f"{name_change}") - logger.warn(name_change) + if purl_to_pod_data_with_soup.get("cocoapods_org_version") is not None: + cocoapods_org_version = purl_to_pod_data_with_soup["cocoapods_org_version"] + elif purl_to_cocoapods_org_url_status.get("cocoapods_org_version") is not None: + cocoapods_org_version = purl_to_cocoapods_org_url_status[ + "cocoapods_org_version" + ] + + if purl_to_pod_data_with_soup.get("cocoapods_org_gh_repo_owner") is not None: + cocoapods_org_gh_repo_owner = purl_to_pod_data_with_soup[ + "cocoapods_org_gh_repo_owner" + ] + elif ( + purl_to_cocoapods_org_url_status.get("cocoapods_org_gh_repo_owner") is not None + ): + cocoapods_org_gh_repo_owner = purl_to_cocoapods_org_url_status[ + "cocoapods_org_gh_repo_owner" + ] + + if purl_to_pod_data_with_soup.get("cocoapods_org_gh_repo_name") is not None: + cocoapods_org_gh_repo_name = purl_to_pod_data_with_soup[ + "cocoapods_org_gh_repo_name" + ] + elif purl_to_cocoapods_org_url_status.get("cocoapods_org_gh_repo_name") is not None: + cocoapods_org_gh_repo_name = purl_to_cocoapods_org_url_status[ + "cocoapods_org_gh_repo_name" + ] api = "https://cdn.cocoapods.org" hashed_path = get_hashed_path(cocoapods_org_pod_name) @@ -518,148 +466,29 @@ def get_cocoapods_data_from_purl(purl): file_prefix = "all_pods_versions_" spec = f"{api}/{file_prefix}{hashed_path_underscore}.txt" data_list = get_cocoapod_tags(spec, cocoapods_org_pod_name) - - print(f"\npod_summary = {json.dumps(pod_summary, indent=4, sort_keys=False)}\n") - if not version: version = cocoapods_org_version - for tag in data_list: if purl.version and tag != purl.version: continue - tag_pkg = construct_cocoapods_package(purl, name, hashed_path, repository_homepage_url, cocoapods_org_gh_repo_owner, cocoapods_org_gh_repo_name, tag, pod_summary) + tag_pkg = construct_cocoapods_package( + purl, + name, + hashed_path, + repository_homepage_url, + cocoapods_org_gh_repo_owner, + cocoapods_org_gh_repo_name, + tag, + cocoapods_org_pod_name + ) + yield tag_pkg if purl.version: break -def get_cocoapod_tags(spec, cocoapods_org_pod_name): - try: - response = get_complete_response(spec) - response.raise_for_status() - data = response.text.strip() - for line in data.splitlines(): - line = line.strip() - if line.startswith(cocoapods_org_pod_name): - data_list = line.split("/") - if data_list[0] == cocoapods_org_pod_name: - data_list.pop(0) - sorted_data_list = sorted( - data_list, - key=lambda x: versions.SemverVersion(x), - reverse=True, - ) - return sorted_data_list - return None - except: - print(f"Error retrieving data from API") - return None - - -def construct_cocoapods_package( - purl, - name, - hashed_path, - repository_homepage_url, - cocoapods_org_gh_repo_owner, - cocoapods_org_gh_repo_name, - tag, - pod_summary, -): - name = name - homepage_url = None - vcs_url = None - github_url = None - bug_tracking_url = None - code_view_url = None - license_data = None - declared_license = None - primary_language = None - - if cocoapods_org_gh_repo_owner and cocoapods_org_gh_repo_name: - name = cocoapods_org_gh_repo_name - namespace = cocoapods_org_gh_repo_owner - base_path = "https://api.github.com/repos" - api_url = f"{base_path}/{namespace}/{name}" - - response = get_github_rest_no_exception(api_url) - - if "Failed to fetch" not in response: - homepage_url = response.get("homepage") - vcs_url = response.get("git_url") - license_data = response.get("license") or {} - declared_license = license_data.get("spdx_id") - primary_language = response.get("language") - - github_url = "https://github.com" - bug_tracking_url = f"{github_url}/{namespace}/{name}/issues" - code_view_url = f"{github_url}/{namespace}/{name}" - - corrected_name = pod_summary["cocoapods_org_pod_name"] - api_url = f"https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/{hashed_path}/{corrected_name}/{tag}/{corrected_name}.podspec.json" - - response = get_json_response(api_url) - if "Failed to fetch" in response: - logger.error(f"{response}") - print(f"{response}") - return - - homepage_url = response.get("homepage") - - lic = response.get("license") - extracted_license_statement = None - if isinstance(lic, dict): - extracted_license_statement = lic - else: - extracted_license_statement = lic - if not declared_license: - declared_license = extracted_license_statement - - source = response.get("source") - vcs_url = None - download_url = None - if isinstance(source, dict): - git_url = source.get("git", "") - http_url = source.get("http", "") - if http_url: - download_url = http_url - pod_summary["http_url"] = http_url - if git_url and not http_url: - vcs_url = git_url - if git_url.endswith(".git"): - gh_path = git_url[:-4] - - corrected_tag = tag - if source.get("tag").startswith("v"): - corrected_tag = source.get("tag") - download_url = f"{gh_path}/archive/refs/tags/{corrected_tag}.tar.gz" - else: - download_url = None - elif git_url: - vcs_url = git_url - elif isinstance(source, str): - if not vcs_url: - vcs_url = source - - purl_pkg = Package( - homepage_url=homepage_url, - api_url=api_url, - bug_tracking_url=bug_tracking_url, - code_view_url=code_view_url, - download_url=download_url, - declared_license=declared_license, - primary_language=primary_language, - repository_homepage_url=repository_homepage_url, - vcs_url=vcs_url, - **purl.to_dict(), - ) - purl_pkg.version = tag - - return purl_pkg - - @dataclasses.dataclass class DirectoryListedSource: source_url: str = dataclasses.field( diff --git a/src/fetchcode/package_util.py b/src/fetchcode/package_util.py index 282f9261..abeaba57 100644 --- a/src/fetchcode/package_util.py +++ b/src/fetchcode/package_util.py @@ -15,13 +15,20 @@ # specific language governing permissions and limitations under the License. import dataclasses +import logging +import os import re import attr +from bs4 import BeautifulSoup +from univers import versions from fetchcode import utils from fetchcode.packagedcode_models import Package +LOG_FILE_LOCATION = os.path.join(os.path.expanduser("~"), "purlcli.log") +logger = logging.getLogger(__name__) + def package_from_dict(package_data): """ @@ -723,3 +730,294 @@ def get_package_info(cls, gh_purl, package_name): "date": "2002-08-19T04:23:00", }, } + + +def get_cocoapods_org_url_status(purl, name, cocoapods_org_url): + purl_to_cocoapods_org_url_status = {} + cocoapods_org_url_head_request = utils.make_head_request(cocoapods_org_url) + cocoapods_org_url_status_code = cocoapods_org_url_head_request.status_code + + if cocoapods_org_url_status_code == 404: + logger.error(f"cocoapods_org_url not found for {name}") + purl_to_cocoapods_org_url_status["return_message"] = "cocoapods_org_url_not_found" + return purl_to_cocoapods_org_url_status + elif cocoapods_org_url_status_code == 503: + logger.error(f"cocoapods_org_url temporarily unavailable for {name}") + purl_to_cocoapods_org_url_status["return_message"] = "cocoapods_org_url_temporarily_unavailable" + return purl_to_cocoapods_org_url_status + elif cocoapods_org_url_status_code == 302: + redirect_url = cocoapods_org_url_head_request.headers['Location'] + redirect_message = f"The cocoapods.org URL {cocoapods_org_url} redirects to {redirect_url}" + logger.warning(redirect_message) + print(redirect_message) + + gh_repo_namespace = None + gh_repo_name = None + if redirect_url.startswith("https://github.com/"): + redirect_url_split = redirect_url.split("/") + if len(redirect_url_split) < 3: + return purl_to_cocoapods_org_url_status + gh_repo_namespace = redirect_url_split[-2] + gh_repo_name = redirect_url_split[-1] + + redirect_to_gh_response = utils.get_complete_response(redirect_url) + if "Failed to fetch" in redirect_to_gh_response: + logger.error(redirect_to_gh_response) + print(redirect_to_gh_response) + purl_to_cocoapods_org_url_status["return_message"] = "failed_to_fetch_github_redirect" + return purl_to_cocoapods_org_url_status + elif "not_found" in redirect_to_gh_response: + redirect_to_gh_not_found = f"Redirect to GitHub not found: {redirect_url}" + logger.error(redirect_to_gh_not_found) + print(redirect_to_gh_not_found) + purl_to_cocoapods_org_url_status["return_message"] = "github_redirect_not_found" + return purl_to_cocoapods_org_url_status + + soup = BeautifulSoup(redirect_to_gh_response.text, "html.parser") + head = soup.find("head") + og_url_tag_get_content = None + corrected_name = None + if head: + og_url_tag = head.find("meta", property="og:url") + if og_url_tag: + og_url = og_url_tag.get("content") + og_url_tag_get_content = og_url + corrected_name = og_url_tag_get_content.split('/')[-1] + else: + no_meta_tag = f"'og:url' meta tag not found in redirect_to_gh_response page for {purl}" + print(no_meta_tag) + logger.error(no_meta_tag) + purl_to_cocoapods_org_url_status["return_message"] = "github_redirect_error" + return purl_to_cocoapods_org_url_status + else: + no_head_section = f"\n section not found in redirect_to_gh_response page for {purl}" + print(no_head_section) + logger.error(no_head_section) + purl_to_cocoapods_org_url_status["return_message"] = "github_redirect_error" + return purl_to_cocoapods_org_url_status + + cocoapods_org_version = None + + purl_to_cocoapods_org_url_status["corrected_name"] = corrected_name + purl_to_cocoapods_org_url_status["cocoapods_org_pod_name"] = corrected_name + purl_to_cocoapods_org_url_status["cocoapods_org_gh_repo_owner"] = gh_repo_namespace + purl_to_cocoapods_org_url_status["cocoapods_org_gh_repo_name"] = gh_repo_name + purl_to_cocoapods_org_url_status["cocoapods_org_version"] = cocoapods_org_version + purl_to_cocoapods_org_url_status["return_message"] = "cocoapods_org_redirects_to_github" + return purl_to_cocoapods_org_url_status + else: + purl_to_cocoapods_org_url_status["return_message"] = "cocoapods_org_url_redirects" + return purl_to_cocoapods_org_url_status + + else: + purl_to_cocoapods_org_url_status["return_message"] = None + return purl_to_cocoapods_org_url_status + + +def get_pod_data_with_soup(purl, name, cocoapods_org_url): + purl_to_pod_data_with_soup = {} + cocoapods_org_response = utils.get_complete_response(cocoapods_org_url) + if "Failed to fetch" in cocoapods_org_response: + logger.error(cocoapods_org_response) + print(cocoapods_org_response) + return + + soup = BeautifulSoup(cocoapods_org_response.text, "html.parser") + cocoapods_org_gh_repo_owner = None + cocoapods_org_gh_repo_name = None + cocoapods_org_gh_repo_url = None + cocoapods_org_podspec_url = None + cocoapods_org_pkg_home_url = None + + for sidebar_links in (soup.find_all('ul', class_ = "links" )): + nested_links = sidebar_links.findChildren("a") + for nested_link in nested_links: + link_text = nested_link.text + link_url = nested_link['href'] + if link_text == 'Homepage': + cocoapods_org_pkg_home_url = link_url + elif link_text == 'GitHub Repo': + split_link = link_url.split('/') + cocoapods_org_gh_repo_owner = split_link[-2] + cocoapods_org_gh_repo_name = split_link[-1] + elif link_text == 'See Podspec': + cocoapods_org_podspec_url = link_url + + if cocoapods_org_gh_repo_owner and cocoapods_org_gh_repo_name: + cocoapods_org_gh_repo_url = f"https://github.com/{cocoapods_org_gh_repo_owner}/{cocoapods_org_gh_repo_name}" + cocoapods_org_gh_repo_url_head_request = utils.make_head_request(cocoapods_org_gh_repo_url) + cocoapods_org_gh_repo_url_status_code = cocoapods_org_gh_repo_url_head_request.status_code + purl_to_pod_data_with_soup["cocoapods_org_gh_repo_url_status_code"] = cocoapods_org_gh_repo_url_status_code + + base_path = "https://api.github.com/repos" + api_url = f"{base_path}/{cocoapods_org_gh_repo_owner}/{cocoapods_org_gh_repo_name}" + github_rest_no_exception_response = utils.get_github_rest_no_exception(api_url) + if "Failed to fetch" in github_rest_no_exception_response: + logger.error(f"{github_rest_no_exception_response}") + print(f"{github_rest_no_exception_response}") + + purl_to_pod_data_with_soup["cocoapods_org_gh_repo_owner"] = cocoapods_org_gh_repo_owner + purl_to_pod_data_with_soup["cocoapods_org_gh_repo_name"] = cocoapods_org_gh_repo_name + purl_to_pod_data_with_soup["cocoapods_org_gh_repo_url"] = cocoapods_org_gh_repo_url + purl_to_pod_data_with_soup["cocoapods_org_podspec_url"] = cocoapods_org_podspec_url + purl_to_pod_data_with_soup["cocoapods_org_pkg_home_url"] = cocoapods_org_pkg_home_url + + if cocoapods_org_gh_repo_owner is None or cocoapods_org_gh_repo_name is None: + no_github_repo = f"No GitHub repo found on cocoapods.org for {name}" + print(f"{no_github_repo}") + logger.warning(no_github_repo) + + if cocoapods_org_podspec_url is None: + no_podspec = f"No podspec found on cocoapods.org for {name}" + print(f"{no_podspec}") + logger.warning(no_podspec) + purl_to_pod_data_with_soup["no_podspec"] = no_podspec + + cocoapods_org_version = None + purl_to_pod_data_with_soup["cocoapods_org_version"] = cocoapods_org_version + if cocoapods_org_podspec_url: + cocoapods_org_version = cocoapods_org_podspec_url.split("/")[-2] + + cocoapods_org_pod_name = None + head = soup.find("head") + if head: + og_title_tag = head.find("meta", property="og:title") + if og_title_tag: + og_title = og_title_tag.get("content") + cocoapods_org_pod_name = og_title + else: + no_meta_tag = f"'og:title' meta tag not found in cocoapods.org page for {purl}" + print(no_meta_tag) + logger.error(no_meta_tag) + else: + no_head_section = f"\n section not found in cocoapods.org page for {purl}" + print(no_head_section) + logger.error(no_head_section) + + purl_to_pod_data_with_soup["cocoapods_org_pod_name"] = cocoapods_org_pod_name + input_name = name + if input_name != cocoapods_org_pod_name: + name_change = (f"Input PURL name '{input_name}' analyzed as '{cocoapods_org_pod_name}' per {cocoapods_org_url}") + input_name = cocoapods_org_pod_name + print(f"{name_change}") + logger.warning(name_change) + + return purl_to_pod_data_with_soup + + +def get_cocoapod_tags(spec, cocoapods_org_pod_name): + try: + response = utils.get_text_response(spec) + data = response.strip() + for line in data.splitlines(): + line = line.strip() + if line.startswith(cocoapods_org_pod_name): + data_list = line.split("/") + if data_list[0] == cocoapods_org_pod_name: + data_list.pop(0) + sorted_data_list = sorted( + data_list, + key=lambda x: versions.SemverVersion(x), + reverse=True, + ) + return sorted_data_list + return None + except: + print(f"Error retrieving cocoapods tag data from cdn.cocoapods.org") + return None + + +def construct_cocoapods_package( + purl, + name, + hashed_path, + repository_homepage_url, + cocoapods_org_gh_repo_owner, + cocoapods_org_gh_repo_name, + tag, + cocoapods_org_pod_name +): + name = name + homepage_url = None + vcs_url = None + github_url = None + bug_tracking_url = None + code_view_url = None + license_data = None + declared_license = None + primary_language = None + + if cocoapods_org_gh_repo_owner and cocoapods_org_gh_repo_name: + name = cocoapods_org_gh_repo_name + namespace = cocoapods_org_gh_repo_owner + base_path = "https://api.github.com/repos" + api_url = f"{base_path}/{namespace}/{name}" + gh_repo_api_response = utils.get_github_rest_no_exception(api_url) + + if "Failed to fetch" not in gh_repo_api_response: + homepage_url = gh_repo_api_response.get("homepage") + vcs_url = gh_repo_api_response.get("git_url") + license_data = gh_repo_api_response.get("license") or {} + declared_license = license_data.get("spdx_id") + primary_language = gh_repo_api_response.get("language") + + github_url = "https://github.com" + bug_tracking_url = f"{github_url}/{namespace}/{name}/issues" + code_view_url = f"{github_url}/{namespace}/{name}" + + corrected_name = cocoapods_org_pod_name + podspec_api_url = f"https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/{hashed_path}/{corrected_name}/{tag}/{corrected_name}.podspec.json" + podspec_api_response = utils.get_json_response(podspec_api_url) + + if "Failed to fetch" in podspec_api_response: + logger.error(f"{podspec_api_response}") + print(f"{podspec_api_response}") + return + + homepage_url = podspec_api_response.get("homepage") + + lic = podspec_api_response.get("license") + extracted_license_statement = None + if isinstance(lic, dict): + extracted_license_statement = lic + else: + extracted_license_statement = lic + if not declared_license: + declared_license = extracted_license_statement + + source = podspec_api_response.get("source") + vcs_url = None + download_url = None + if isinstance(source, dict): + git_url = source.get("git", "") + http_url = source.get("http", "") + if http_url: + download_url = http_url + if git_url and not http_url: + if git_url.endswith(".git") and "github" in git_url: + gh_path = git_url[:-4] + corrected_tag = tag + if source.get("tag") and source.get("tag").startswith("v"): + corrected_tag = source.get("tag") + download_url = f"{gh_path}/archive/refs/tags/{corrected_tag}.tar.gz" + vcs_url = git_url + elif git_url: + vcs_url = git_url + elif isinstance(source, str): + if not vcs_url: + vcs_url = source + + purl_pkg = Package( + homepage_url=homepage_url, + api_url=podspec_api_url, + bug_tracking_url=bug_tracking_url, + code_view_url=code_view_url, + download_url=download_url, + declared_license=declared_license, + primary_language=primary_language, + repository_homepage_url=repository_homepage_url, + vcs_url=vcs_url, + **purl.to_dict(), + ) + purl_pkg.version = tag + return purl_pkg diff --git a/src/fetchcode/utils.py b/src/fetchcode/utils.py index 46a626bb..b0edfde2 100644 --- a/src/fetchcode/utils.py +++ b/src/fetchcode/utils.py @@ -202,10 +202,15 @@ def get_json_response(url, headers=None): return f"Failed to fetch: {url}" +def get_text_response(url, headers=None): + resp = requests.get(url, headers=headers) + if resp.status_code == 200: + return resp.text + + return f"Failed to fetch: {url}" + + def get_complete_response(url, headers=None, params=None): - """ - Generate `Package` object for a `url` string - """ resp = requests.get(url, headers=headers, params=params) if resp.status_code == 200: return resp @@ -216,9 +221,6 @@ def get_complete_response(url, headers=None, params=None): def make_head_request(url, headers=None): - """ - Check whether the URL status code is 200 or not. - """ try: resp = requests.head(url, headers=headers) diff --git a/tests/data/cocoapods/AFNetworking.html b/tests/data/cocoapods/AFNetworking.html new file mode 100644 index 00000000..501fd2a9 --- /dev/null +++ b/tests/data/cocoapods/AFNetworking.html @@ -0,0 +1,721 @@ + + + + + AFNetworking on CocoaPods.org + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+

AFNetworking 4.0.1

+
+
+
+
+
+
+
+
+
+

AFNetworking 4.0.1

+
×
+

+ + + + + + + + + + + + + + + + + + + +
TestsTested + ✓
LangLanguage + Obj-CObjective C +
License MIT +
ReleasedLast Release + Jan 2023
+

Maintained by Kyle Fuller, Mattt, Jeff Kelley, Jon Shier, Kevin Harwood, Christian Noon.

+
+ +
+ +
+
+
+

AFNetworking 4.0.1

+
+ + +
+
This library is deprecated, it is recommended you look at Alamofire.
+ +
+
+
+
+

+ AFNetworking +

+

Build Status + CocoaPods Compatible + Carthage Compatible + Platform + Twitter +

+

AFNetworking is a delightful networking library for iOS, macOS, watchOS, and tvOS. It's built on top of the Foundation URL Loading System, extending the powerful high-level networking abstractions built into Cocoa. It has a modular architecture with well-designed, feature-rich APIs that are a joy to use.

+

Perhaps the most important feature of all, however, is the amazing community of developers who use and contribute to AFNetworking every day. AFNetworking powers some of the most popular and critically-acclaimed apps on the iPhone, iPad, and Mac.

+

How To Get Started

+ +

Communication

+
    +
  • If you need help, use Stack Overflow. (Tag 'afnetworking')
    • +
  • If you'd like to ask a general question, use Stack Overflow.
    • +
  • If you found a bug, and can provide steps to reliably reproduce it, open an issue.
    • +
  • If you have a feature request, open an issue.
    • +
  • If you want to contribute, submit a pull request.
    • +
+

Installation

+

AFNetworking supports multiple methods for installing the library in a project.

+

Installation with CocoaPods

+

To integrate AFNetworking into your Xcode project using CocoaPods, specify it in your Podfile:

+
+ 
pod 'AFNetworking', '~> 4.0'
+
+

Installation with Swift Package Manager

+

Once you have your Swift package set up, adding AFNetworking as a dependency is as easy as adding it to the dependencies value of your Package.swift.

+
+ 
dependencies: [
+    .package(url: "https://github.com/AFNetworking/AFNetworking.git", .upToNextMajor(from: "4.0.0"))
+]
+
+
+

Note: AFNetworking's Swift package does not include it's UIKit extensions.

+
+

Installation with Carthage

+

Carthage is a decentralized dependency manager that builds your dependencies and provides you with binary frameworks. To integrate AFNetworking, add the following to your Cartfile.

+
+ 
github "AFNetworking/AFNetworking" ~> 4.0
+
+
+

Requirements

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
AFNetworking VersionMinimum iOS TargetMinimum macOS TargetMinimum watchOS TargetMinimum tvOS TargetNotes
4.xiOS 9macOS 10.10watchOS 2.0tvOS 9.0Xcode 11+ is required.
3.xiOS 7OS X 10.9watchOS 2.0tvOS 9.0Xcode 7+ is required. NSURLConnectionOperation support has been removed.
2.6 -> 2.6.3iOS 7OS X 10.9watchOS 2.0n/aXcode 7+ is required.
2.0 -> 2.5.4iOS 6OS X 10.8n/an/aXcode 5+ is required. NSURLSession subspec requires iOS 7 or OS X 10.9.
1.xiOS 5Mac OS X 10.7n/an/a
0.10.xiOS 4Mac OS X 10.6n/an/a
+

(macOS projects must support 64-bit with modern Cocoa runtime).

+
+

Programming in Swift? Try Alamofire for a more conventional set of APIs.

+
+

Architecture

+

NSURLSession

+
    +
  • AFURLSessionManager
    • +
  • AFHTTPSessionManager
    • +
+

Serialization

+
    +
  • <AFURLRequestSerialization> +
      +
    • AFHTTPRequestSerializer
      • +
    • AFJSONRequestSerializer
      • +
    • AFPropertyListRequestSerializer
      • +
    +
    • +
  • <AFURLResponseSerialization> +
      +
    • AFHTTPResponseSerializer
      • +
    • AFJSONResponseSerializer
      • +
    • AFXMLParserResponseSerializer
      • +
    • AFXMLDocumentResponseSerializer (macOS)
      • +
    • AFPropertyListResponseSerializer
      • +
    • AFImageResponseSerializer
      • +
    • AFCompoundResponseSerializer
      • +
    +
    • +
+

Additional Functionality

+
    +
  • AFSecurityPolicy
    • +
  • AFNetworkReachabilityManager
    • +
+

Usage

+

AFURLSessionManager

+

AFURLSessionManager creates and manages an NSURLSession object based on a specified NSURLSessionConfiguration object, which conforms to <NSURLSessionTaskDelegate>, <NSURLSessionDataDelegate>, <NSURLSessionDownloadDelegate>, and <NSURLSessionDelegate>.

+

Creating a Download Task

+
+ 
NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration defaultSessionConfiguration];
+AFURLSessionManager *manager = [[AFURLSessionManager alloc] initWithSessionConfiguration:configuration];
+
+NSURL *URL = [NSURL URLWithString:@"http://example.com/download.zip"];
+NSURLRequest *request = [NSURLRequest requestWithURL:URL];
+
+NSURLSessionDownloadTask *downloadTask = [manager downloadTaskWithRequest:request progress:nil destination:^NSURL *(NSURL *targetPath, NSURLResponse *response) {
+    NSURL *documentsDirectoryURL = [[NSFileManager defaultManager] URLForDirectory:NSDocumentDirectory inDomain:NSUserDomainMask appropriateForURL:nil create:NO error:nil];
+    return [documentsDirectoryURL URLByAppendingPathComponent:[response suggestedFilename]];
+} completionHandler:^(NSURLResponse *response, NSURL *filePath, NSError *error) {
+    NSLog(@"File downloaded to: %@", filePath);
+}];
+[downloadTask resume];
+
+

Creating an Upload Task

+
+ 
NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration defaultSessionConfiguration];
+AFURLSessionManager *manager = [[AFURLSessionManager alloc] initWithSessionConfiguration:configuration];
+
+NSURL *URL = [NSURL URLWithString:@"http://example.com/upload"];
+NSURLRequest *request = [NSURLRequest requestWithURL:URL];
+
+NSURL *filePath = [NSURL fileURLWithPath:@"file://path/to/image.png"];
+NSURLSessionUploadTask *uploadTask = [manager uploadTaskWithRequest:request fromFile:filePath progress:nil completionHandler:^(NSURLResponse *response, id responseObject, NSError *error) {
+    if (error) {
+        NSLog(@"Error: %@", error);
+    } else {
+        NSLog(@"Success: %@ %@", response, responseObject);
+    }
+}];
+[uploadTask resume];
+
+

Creating an Upload Task for a Multi-Part Request, with Progress

+
+ 
NSMutableURLRequest *request = [[AFHTTPRequestSerializer serializer] multipartFormRequestWithMethod:@"POST" URLString:@"http://example.com/upload" parameters:nil constructingBodyWithBlock:^(id<AFMultipartFormData> formData) {
+        [formData appendPartWithFileURL:[NSURL fileURLWithPath:@"file://path/to/image.jpg"] name:@"file" fileName:@"filename.jpg" mimeType:@"image/jpeg" error:nil];
+    } error:nil];
+
+AFURLSessionManager *manager = [[AFURLSessionManager alloc] initWithSessionConfiguration:[NSURLSessionConfiguration defaultSessionConfiguration]];
+
+NSURLSessionUploadTask *uploadTask;
+uploadTask = [manager
+              uploadTaskWithStreamedRequest:request
+              progress:^(NSProgress * _Nonnull uploadProgress) {
+                  // This is not called back on the main queue.
+                  // You are responsible for dispatching to the main queue for UI updates
+                  dispatch_async(dispatch_get_main_queue(), ^{
+                      //Update the progress view
+                      [progressView setProgress:uploadProgress.fractionCompleted];
+                  });
+              }
+              completionHandler:^(NSURLResponse * _Nonnull response, id  _Nullable responseObject, NSError * _Nullable error) {
+                  if (error) {
+                      NSLog(@"Error: %@", error);
+                  } else {
+                      NSLog(@"%@ %@", response, responseObject);
+                  }
+              }];
+
+[uploadTask resume];
+
+

Creating a Data Task

+
+ 
NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration defaultSessionConfiguration];
+AFURLSessionManager *manager = [[AFURLSessionManager alloc] initWithSessionConfiguration:configuration];
+
+NSURL *URL = [NSURL URLWithString:@"http://httpbin.org/get"];
+NSURLRequest *request = [NSURLRequest requestWithURL:URL];
+
+NSURLSessionDataTask *dataTask = [manager dataTaskWithRequest:request completionHandler:^(NSURLResponse *response, id responseObject, NSError *error) {
+    if (error) {
+        NSLog(@"Error: %@", error);
+    } else {
+        NSLog(@"%@ %@", response, responseObject);
+    }
+}];
+[dataTask resume];
+
+
+

Request Serialization

+

Request serializers create requests from URL strings, encoding parameters as either a query string or HTTP body.

+
+ 
NSString *URLString = @"http://example.com";
+NSDictionary *parameters = @{@"foo": @"bar", @"baz": @[@1, @2, @3]};
+
+

Query String Parameter Encoding

+
+ 
[[AFHTTPRequestSerializer serializer] requestWithMethod:@"GET" URLString:URLString parameters:parameters error:nil];
+
+
+ 
GET http://example.com?foo=bar&baz[]=1&baz[]=2&baz[]=3
+
+
+

URL Form Parameter Encoding

+
+ 
[[AFHTTPRequestSerializer serializer] requestWithMethod:@"POST" URLString:URLString parameters:parameters error:nil];
+
+
+ 
POST http://example.com/
+Content-Type: application/x-www-form-urlencoded
+
+foo=bar&baz[]=1&baz[]=2&baz[]=3
+
+
+

JSON Parameter Encoding

+
+ 
[[AFJSONRequestSerializer serializer] requestWithMethod:@"POST" URLString:URLString parameters:parameters error:nil];
+
+
+ 
POST http://example.com/
+Content-Type: application/json
+
+{"foo": "bar", "baz": [1,2,3]}
+
+
+
+

Network Reachability Manager

+

AFNetworkReachabilityManager monitors the reachability of domains, and addresses for both WWAN and WiFi network interfaces.

+
    +
  • Do not use Reachability to determine if the original request should be sent. +
      +
    • You should try to send it.
      • +
    +
    • +
  • You can use Reachability to determine when a request should be automatically retried. +
      +
    • Although it may still fail, a Reachability notification that the connectivity is available is a good time to retry something.
      • +
    +
    • +
  • Network reachability is a useful tool for determining why a request might have failed. +
      +
    • After a network request has failed, telling the user they're offline is better than giving them a more technical but accurate error, such as "request timed out."
      • +
    +
    • +
+

See also WWDC 2012 session 706, "Networking Best Practices.".

+

Shared Network Reachability

+
+ 
[[AFNetworkReachabilityManager sharedManager] setReachabilityStatusChangeBlock:^(AFNetworkReachabilityStatus status) {
+    NSLog(@"Reachability: %@", AFStringFromNetworkReachabilityStatus(status));
+}];
+
+[[AFNetworkReachabilityManager sharedManager] startMonitoring];
+
+
+

Security Policy

+

AFSecurityPolicy evaluates server trust against pinned X.509 certificates and public keys over secure connections.

+

Adding pinned SSL certificates to your app helps prevent man-in-the-middle attacks and other vulnerabilities. Applications dealing with sensitive customer data or financial information are strongly encouraged to route all communication over an HTTPS connection with SSL pinning configured and enabled.

+

Allowing Invalid SSL Certificates

+
+ 
AFHTTPSessionManager *manager = [AFHTTPSessionManager manager];
+manager.securityPolicy.allowInvalidCertificates = YES; // not recommended for production
+
+
+

Unit Tests

+

AFNetworking includes a suite of unit tests within the Tests subdirectory. These tests can be run simply be executed the test action on the platform framework you would like to test.

+

Credits

+

AFNetworking is owned and maintained by the Alamofire Software Foundation.

+

AFNetworking was originally created by Scott Raymond and Mattt Thompson in the development of Gowalla for iPhone.

+

AFNetworking's logo was designed by Alan Defibaugh.

+

And most of all, thanks to AFNetworking's growing list of contributors.

+

Security Disclosure

+

If you believe you have identified a security vulnerability with AFNetworking, you should report it as soon as possible via email to [email protected]. Please do not post it to a public issue tracker.

+

License

+

AFNetworking is released under the MIT license. See LICENSE for details.

+
+
+
+
+
+
+
+
+
+
+
+ + +
+ + + + + diff --git a/tests/data/cocoapods/afnetworking_github_rest_no_exception_response.json b/tests/data/cocoapods/afnetworking_github_rest_no_exception_response.json new file mode 100644 index 00000000..6ffc8b78 --- /dev/null +++ b/tests/data/cocoapods/afnetworking_github_rest_no_exception_response.json @@ -0,0 +1,137 @@ +{ + "id": 1828795, + "node_id": "MDEwOlJlcG9zaXRvcnkxODI4Nzk1", + "name": "AFNetworking", + "full_name": "AFNetworking/AFNetworking", + "private": false, + "owner": { + "login": "AFNetworking", + "id": 1181541, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjExODE1NDE=", + "avatar_url": "https: //avatars.githubusercontent.com/u/1181541?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/AFNetworking", + "html_url": "https://github.com/AFNetworking", + "followers_url": "https://api.github.com/users/AFNetworking/followers", + "following_url": "https://api.github.com/users/AFNetworking/following{/other_user}", + "gists_url": "https://api.github.com/users/AFNetworking/gists{/gist_id}", + "starred_url": "https://api.github.com/users/AFNetworking/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/AFNetworking/subscriptions", + "organizations_url": "https://api.github.com/users/AFNetworking/orgs", + "repos_url": "https://api.github.com/users/AFNetworking/repos", + "events_url": "https://api.github.com/users/AFNetworking/events{/privacy}", + "received_events_url": "https://api.github.com/users/AFNetworking/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/AFNetworking/AFNetworking", + "description": "A delightful networking framework for iOS, macOS, watchOS, and tvOS.", + "fork": false, + "url": "https://api.github.com/repos/AFNetworking/AFNetworking", + "forks_url": "https://api.github.com/repos/AFNetworking/AFNetworking/forks", + "keys_url": "https://api.github.com/repos/AFNetworking/AFNetworking/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/AFNetworking/AFNetworking/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/AFNetworking/AFNetworking/teams", + "hooks_url": "https://api.github.com/repos/AFNetworking/AFNetworking/hooks", + "issue_events_url": "https://api.github.com/repos/AFNetworking/AFNetworking/issues/events{/number}", + "events_url": "https://api.github.com/repos/AFNetworking/AFNetworking/events", + "assignees_url": "https://api.github.com/repos/AFNetworking/AFNetworking/assignees{/user}", + "branches_url": "https://api.github.com/repos/AFNetworking/AFNetworking/branches{/branch}", + "tags_url": "https://api.github.com/repos/AFNetworking/AFNetworking/tags", + "blobs_url": "https://api.github.com/repos/AFNetworking/AFNetworking/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/AFNetworking/AFNetworking/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/AFNetworking/AFNetworking/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/AFNetworking/AFNetworking/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/AFNetworking/AFNetworking/statuses/{sha}", + "languages_url": "https://api.github.com/repos/AFNetworking/AFNetworking/languages", + "stargazers_url": "https://api.github.com/repos/AFNetworking/AFNetworking/stargazers", + "contributors_url": "https://api.github.com/repos/AFNetworking/AFNetworking/contributors", + "subscribers_url": "https://api.github.com/repos/AFNetworking/AFNetworking/subscribers", + "subscription_url": "https://api.github.com/repos/AFNetworking/AFNetworking/subscription", + "commits_url": "https://api.github.com/repos/AFNetworking/AFNetworking/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/AFNetworking/AFNetworking/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/AFNetworking/AFNetworking/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/AFNetworking/AFNetworking/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/AFNetworking/AFNetworking/contents/{+path}", + "compare_url": "https://api.github.com/repos/AFNetworking/AFNetworking/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/AFNetworking/AFNetworking/merges", + "archive_url": "https://api.github.com/repos/AFNetworking/AFNetworking/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/AFNetworking/AFNetworking/downloads", + "issues_url": "https://api.github.com/repos/AFNetworking/AFNetworking/issues{/number}", + "pulls_url": "https://api.github.com/repos/AFNetworking/AFNetworking/pulls{/number}", + "milestones_url": "https://api.github.com/repos/AFNetworking/AFNetworking/milestones{/number}", + "notifications_url": "https://api.github.com/repos/AFNetworking/AFNetworking/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/AFNetworking/AFNetworking/labels{/name}", + "releases_url": "https://api.github.com/repos/AFNetworking/AFNetworking/releases{/id}", + "deployments_url": "https://api.github.com/repos/AFNetworking/AFNetworking/deployments", + "created_at": "2011-05-31T21:28:44Z", + "updated_at": "2024-05-02T12:19:46Z", + "pushed_at": "2023-01-17T19:30:05Z", + "git_url": "git://github.com/AFNetworking/AFNetworking.git", + "ssh_url": "git@github.com:AFNetworking/AFNetworking.git", + "clone_url": "https://github.com/AFNetworking/AFNetworking.git", + "svn_url": "https://github.com/AFNetworking/AFNetworking", + "homepage": "http://afnetworking.com", + "size": 6129, + "stargazers_count": 33331, + "watchers_count": 33331, + "language": "Objective-C", + "has_issues": false, + "has_projects": false, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "has_discussions": false, + "forks_count": 10363, + "mirror_url": null, + "archived": true, + "disabled": false, + "open_issues_count": 103, + "license": { + "key": "mit", + "name": "MIT License", + "spdx_id": "MIT", + "url": "https://api.github.com/licenses/mit", + "node_id": "MDc6TGljZW5zZTEz" + }, + "allow_forking": true, + "is_template": false, + "web_commit_signoff_required": false, + "topics": [], + "visibility": "public", + "forks": 10363, + "open_issues": 103, + "watchers": 33331, + "default_branch": "master", + "permissions": { + "admin": false, + "maintain": false, + "push": false, + "triage": false, + "pull": true + }, + "temp_clone_token": "", + "custom_properties": {}, + "organization": { + "login": "AFNetworking", + "id": 1181541, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjExODE1NDE=", + "avatar_url": "https://avatars.githubusercontent.com/u/1181541?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/AFNetworking", + "html_url": "https://github.com/AFNetworking", + "followers_url": "https://api.github.com/users/AFNetworking/followers", + "following_url": "https://api.github.com/users/AFNetworking/following{/other_user}", + "gists_url": "https://api.github.com/users/AFNetworking/gists{/gist_id}", + "starred_url": "https://api.github.com/users/AFNetworking/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/AFNetworking/subscriptions", + "organizations_url": "https://api.github.com/users/AFNetworking/orgs", + "repos_url": "https://api.github.com/users/AFNetworking/repos", + "events_url": "https://api.github.com/users/AFNetworking/events{/privacy}", + "received_events_url": "https://api.github.com/users/AFNetworking/received_events", + "type": "Organization", + "site_admin": false + }, + "network_count": 10363, + "subscribers_count": 1590 +} diff --git a/tests/data/cocoapods/afnetworking_response_text.txt b/tests/data/cocoapods/afnetworking_response_text.txt new file mode 100644 index 00000000..1f2b81f0 --- /dev/null +++ b/tests/data/cocoapods/afnetworking_response_text.txt @@ -0,0 +1,445 @@ +AFNetworking on CocoaPods.org

AFNetworking 4.0.1

AFNetworking 4.0.1

×

TestsTested +✓
LangLanguage + Obj-CObjective C +
License MIT +
ReleasedLast Release +Jan 2023

Maintained by Kyle Fuller, Mattt, Jeff Kelley, Jon Shier, Kevin Harwood, Christian Noon.

AFNetworking 4.0.1

This library is deprecated, it is recommended you look at Alamofire.

+AFNetworking +

+

Build Status +CocoaPods Compatible +Carthage Compatible +Platform +Twitter

+

AFNetworking is a delightful networking library for iOS, macOS, watchOS, and tvOS. It's built on top of the Foundation URL Loading System, extending the powerful high-level networking abstractions built into Cocoa. It has a modular architecture with well-designed, feature-rich APIs that are a joy to use.

+

Perhaps the most important feature of all, however, is the amazing community of developers who use and contribute to AFNetworking every day. AFNetworking powers some of the most popular and critically-acclaimed apps on the iPhone, iPad, and Mac.

+

How To Get Started

+ +

Communication

+
    +
  • If you need help, use Stack Overflow. (Tag 'afnetworking')
    • +
  • If you'd like to ask a general question, use Stack Overflow.
    • +
  • If you found a bug, and can provide steps to reliably reproduce it, open an issue.
    • +
  • If you have a feature request, open an issue.
    • +
  • If you want to contribute, submit a pull request.
    • +
+

Installation

+

AFNetworking supports multiple methods for installing the library in a project.

+

Installation with CocoaPods

+

To integrate AFNetworking into your Xcode project using CocoaPods, specify it in your Podfile:

+
pod 'AFNetworking', '~> 4.0'
+

Installation with Swift Package Manager

+

Once you have your Swift package set up, adding AFNetworking as a dependency is as easy as adding it to the dependencies value of your Package.swift.

+
dependencies: [
+    .package(url: "https://github.com/AFNetworking/AFNetworking.git", .upToNextMajor(from: "4.0.0"))
+]
+
+

Note: AFNetworking's Swift package does not include it's UIKit extensions.

+
+

Installation with Carthage

+

Carthage is a decentralized dependency manager that builds your dependencies and provides you with binary frameworks. To integrate AFNetworking, add the following to your Cartfile.

+
github "AFNetworking/AFNetworking" ~> 4.0
+
+

Requirements

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
AFNetworking VersionMinimum iOS TargetMinimum macOS TargetMinimum watchOS TargetMinimum tvOS TargetNotes
4.xiOS 9macOS 10.10watchOS 2.0tvOS 9.0Xcode 11+ is required.
3.xiOS 7OS X 10.9watchOS 2.0tvOS 9.0Xcode 7+ is required. NSURLConnectionOperation support has been removed.
2.6 -> 2.6.3iOS 7OS X 10.9watchOS 2.0n/aXcode 7+ is required.
2.0 -> 2.5.4iOS 6OS X 10.8n/an/aXcode 5+ is required. NSURLSession subspec requires iOS 7 or OS X 10.9.
1.xiOS 5Mac OS X 10.7n/an/a
0.10.xiOS 4Mac OS X 10.6n/an/a
+

(macOS projects must support 64-bit with modern Cocoa runtime).

+
+

Programming in Swift? Try Alamofire for a more conventional set of APIs.

+
+

Architecture

+

NSURLSession

+
    +
  • AFURLSessionManager
    • +
  • AFHTTPSessionManager
    • +
+

Serialization

+
    +
  • <AFURLRequestSerialization> +
      +
    • AFHTTPRequestSerializer
      • +
    • AFJSONRequestSerializer
      • +
    • AFPropertyListRequestSerializer
      • +
    +
    • +
  • <AFURLResponseSerialization> +
      +
    • AFHTTPResponseSerializer
      • +
    • AFJSONResponseSerializer
      • +
    • AFXMLParserResponseSerializer
      • +
    • AFXMLDocumentResponseSerializer (macOS)
      • +
    • AFPropertyListResponseSerializer
      • +
    • AFImageResponseSerializer
      • +
    • AFCompoundResponseSerializer
      • +
    +
    • +
+

Additional Functionality

+
    +
  • AFSecurityPolicy
    • +
  • AFNetworkReachabilityManager
    • +
+

Usage

+

AFURLSessionManager

+

AFURLSessionManager creates and manages an NSURLSession object based on a specified NSURLSessionConfiguration object, which conforms to <NSURLSessionTaskDelegate>, <NSURLSessionDataDelegate>, <NSURLSessionDownloadDelegate>, and <NSURLSessionDelegate>.

+

Creating a Download Task

+
NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration defaultSessionConfiguration];
+AFURLSessionManager *manager = [[AFURLSessionManager alloc] initWithSessionConfiguration:configuration];
+
+NSURL *URL = [NSURL URLWithString:@"http://example.com/download.zip"];
+NSURLRequest *request = [NSURLRequest requestWithURL:URL];
+
+NSURLSessionDownloadTask *downloadTask = [manager downloadTaskWithRequest:request progress:nil destination:^NSURL *(NSURL *targetPath, NSURLResponse *response) {
+    NSURL *documentsDirectoryURL = [[NSFileManager defaultManager] URLForDirectory:NSDocumentDirectory inDomain:NSUserDomainMask appropriateForURL:nil create:NO error:nil];
+    return [documentsDirectoryURL URLByAppendingPathComponent:[response suggestedFilename]];
+} completionHandler:^(NSURLResponse *response, NSURL *filePath, NSError *error) {
+    NSLog(@"File downloaded to: %@", filePath);
+}];
+[downloadTask resume];
+

Creating an Upload Task

+
NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration defaultSessionConfiguration];
+AFURLSessionManager *manager = [[AFURLSessionManager alloc] initWithSessionConfiguration:configuration];
+
+NSURL *URL = [NSURL URLWithString:@"http://example.com/upload"];
+NSURLRequest *request = [NSURLRequest requestWithURL:URL];
+
+NSURL *filePath = [NSURL fileURLWithPath:@"file://path/to/image.png"];
+NSURLSessionUploadTask *uploadTask = [manager uploadTaskWithRequest:request fromFile:filePath progress:nil completionHandler:^(NSURLResponse *response, id responseObject, NSError *error) {
+    if (error) {
+        NSLog(@"Error: %@", error);
+    } else {
+        NSLog(@"Success: %@ %@", response, responseObject);
+    }
+}];
+[uploadTask resume];
+

Creating an Upload Task for a Multi-Part Request, with Progress

+
NSMutableURLRequest *request = [[AFHTTPRequestSerializer serializer] multipartFormRequestWithMethod:@"POST" URLString:@"http://example.com/upload" parameters:nil constructingBodyWithBlock:^(id<AFMultipartFormData> formData) {
+        [formData appendPartWithFileURL:[NSURL fileURLWithPath:@"file://path/to/image.jpg"] name:@"file" fileName:@"filename.jpg" mimeType:@"image/jpeg" error:nil];
+    } error:nil];
+
+AFURLSessionManager *manager = [[AFURLSessionManager alloc] initWithSessionConfiguration:[NSURLSessionConfiguration defaultSessionConfiguration]];
+
+NSURLSessionUploadTask *uploadTask;
+uploadTask = [manager
+              uploadTaskWithStreamedRequest:request
+              progress:^(NSProgress * _Nonnull uploadProgress) {
+                  // This is not called back on the main queue.
+                  // You are responsible for dispatching to the main queue for UI updates
+                  dispatch_async(dispatch_get_main_queue(), ^{
+                      //Update the progress view
+                      [progressView setProgress:uploadProgress.fractionCompleted];
+                  });
+              }
+              completionHandler:^(NSURLResponse * _Nonnull response, id  _Nullable responseObject, NSError * _Nullable error) {
+                  if (error) {
+                      NSLog(@"Error: %@", error);
+                  } else {
+                      NSLog(@"%@ %@", response, responseObject);
+                  }
+              }];
+
+[uploadTask resume];
+

Creating a Data Task

+
NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration defaultSessionConfiguration];
+AFURLSessionManager *manager = [[AFURLSessionManager alloc] initWithSessionConfiguration:configuration];
+
+NSURL *URL = [NSURL URLWithString:@"http://httpbin.org/get"];
+NSURLRequest *request = [NSURLRequest requestWithURL:URL];
+
+NSURLSessionDataTask *dataTask = [manager dataTaskWithRequest:request completionHandler:^(NSURLResponse *response, id responseObject, NSError *error) {
+    if (error) {
+        NSLog(@"Error: %@", error);
+    } else {
+        NSLog(@"%@ %@", response, responseObject);
+    }
+}];
+[dataTask resume];
+
+

Request Serialization

+

Request serializers create requests from URL strings, encoding parameters as either a query string or HTTP body.

+
NSString *URLString = @"http://example.com";
+NSDictionary *parameters = @{@"foo": @"bar", @"baz": @[@1, @2, @3]};
+

Query String Parameter Encoding

+
[[AFHTTPRequestSerializer serializer] requestWithMethod:@"GET" URLString:URLString parameters:parameters error:nil];
+
GET http://example.com?foo=bar&baz[]=1&baz[]=2&baz[]=3
+
+

URL Form Parameter Encoding

+
[[AFHTTPRequestSerializer serializer] requestWithMethod:@"POST" URLString:URLString parameters:parameters error:nil];
+
POST http://example.com/
+Content-Type: application/x-www-form-urlencoded
+
+foo=bar&baz[]=1&baz[]=2&baz[]=3
+
+

JSON Parameter Encoding

+
[[AFJSONRequestSerializer serializer] requestWithMethod:@"POST" URLString:URLString parameters:parameters error:nil];
+
POST http://example.com/
+Content-Type: application/json
+
+{"foo": "bar", "baz": [1,2,3]}
+
+
+

Network Reachability Manager

+

AFNetworkReachabilityManager monitors the reachability of domains, and addresses for both WWAN and WiFi network interfaces.

+
    +
  • Do not use Reachability to determine if the original request should be sent. +
      +
    • You should try to send it.
      • +
    +
    • +
  • You can use Reachability to determine when a request should be automatically retried. +
      +
    • Although it may still fail, a Reachability notification that the connectivity is available is a good time to retry something.
      • +
    +
    • +
  • Network reachability is a useful tool for determining why a request might have failed. +
      +
    • After a network request has failed, telling the user they're offline is better than giving them a more technical but accurate error, such as "request timed out."
      • +
    +
    • +
+

See also WWDC 2012 session 706, "Networking Best Practices.".

+

Shared Network Reachability

+
[[AFNetworkReachabilityManager sharedManager] setReachabilityStatusChangeBlock:^(AFNetworkReachabilityStatus status) {
+    NSLog(@"Reachability: %@", AFStringFromNetworkReachabilityStatus(status));
+}];
+
+[[AFNetworkReachabilityManager sharedManager] startMonitoring];
+
+

Security Policy

+

AFSecurityPolicy evaluates server trust against pinned X.509 certificates and public keys over secure connections.

+

Adding pinned SSL certificates to your app helps prevent man-in-the-middle attacks and other vulnerabilities. Applications dealing with sensitive customer data or financial information are strongly encouraged to route all communication over an HTTPS connection with SSL pinning configured and enabled.

+

Allowing Invalid SSL Certificates

+
AFHTTPSessionManager *manager = [AFHTTPSessionManager manager];
+manager.securityPolicy.allowInvalidCertificates = YES; // not recommended for production
+
+

Unit Tests

+

AFNetworking includes a suite of unit tests within the Tests subdirectory. These tests can be run simply be executed the test action on the platform framework you would like to test.

+

Credits

+

AFNetworking is owned and maintained by the Alamofire Software Foundation.

+

AFNetworking was originally created by Scott Raymond and Mattt Thompson in the development of Gowalla for iPhone.

+

AFNetworking's logo was designed by Alan Defibaugh.

+

And most of all, thanks to AFNetworking's growing list of contributors.

+

Security Disclosure

+

If you believe you have identified a security vulnerability with AFNetworking, you should report it as soon as possible via email to [email protected]. Please do not post it to a public issue tracker.

+

License

+

AFNetworking is released under the MIT license. See LICENSE for details.

+
diff --git a/tests/data/cocoapods/cocoapod_all_pods_versions_5_1_f.txt b/tests/data/cocoapods/cocoapod_all_pods_versions_5_1_f.txt new file mode 100644 index 00000000..82e5262f --- /dev/null +++ b/tests/data/cocoapods/cocoapod_all_pods_versions_5_1_f.txt @@ -0,0 +1,18 @@ +AVOSCloud-tvOS/3.1.7/3.1.7.1/3.2.1/3.2.12/3.2.2/3.2.3/3.2.4/3.2.4.1/3.2.5/3.2.5.1/3.2.6/3.2.7/3.2.8/3.3/3.3.1/3.3.2/3.3.3/3.3.4/3.3.5 +Batch/1.10.0/1.10.1/1.10.2/1.10.3/1.10.4/1.11.0/1.12.0/1.13.0/1.13.1/1.13.2/1.14.0/1.14.1/1.14.2/1.15.0/1.15.1/1.15.2/1.16.0/1.16.1/1.17.0/1.17.1/1.17.2/1.17.3/1.18.0/1.18.1/1.18.2/1.19.0/1.19.1/1.19.2/1.19.3/1.19.4/1.19.5/1.2.3/1.2.4/1.2.5/1.2.6/1.20.0/1.20.0-emailing.1/1.21.0/1.21.1/1.21.2/1.3/1.3.1/1.3.2/1.3.2-2/1.3.2-3/1.4/1.5/1.5.1/1.5.3/1.5.4/1.6.0/1.7.0/1.7.1/1.7.2/1.7.3/1.7.4/1.8.0/1.9.0 +DeptFlow/0.1.0/0.1.1/0.2.0/0.3.0 +HCImagePickerMoudle/0.0.1 +JUtilities/1.0/1.1 +JXFMDBMOperator/1.0.4 +KVLLibraries/1.0.0/1.1.0 +MLMS3Upload/0.1.0/0.2.0/0.3.0/0.4.0 +OptimizelySDKiOS/0.1.1/0.1.2/0.1.3/0.1.4/0.1.5/0.1.6/0.1.7/0.1.8/0.1.9/0.2.1/0.2.1-alpha1/0.2.1-alpha2/0.2.1-alpha3/0.2.1-alpha4/0.3.0/0.3.0-alpha1/0.4.0-alpha1/0.4.0-alpha2/0.4.0-alpha3/0.5.0/0.5.0-alpha1/1.0.0/1.0.1/1.0.1-alpha1/1.1.0/1.1.3/1.1.5/1.1.6/1.1.7/1.1.8/1.1.9/1.3.0/1.4.0/1.5.0/1.5.0-RC/1.5.1/1.5.2/2.0.2-beta1/2.0.2-beta2/2.0.2-beta3/2.0.2-beta4/2.1.0/2.1.1/2.1.2/2.1.3/2.1.4/2.1.5/2.1.6/3.0.0/3.0.1/3.0.2/3.1.0/3.1.1/3.1.2/3.1.3/3.1.4/3.1.5 +QAlone/0.1.2/0.1.3/0.1.8/0.1.9/0.2.1/0.2.2/0.2.3/0.2.4/0.3.8/0.3.9/0.4.1/0.4.2/0.4.3/0.4.4/0.4.5/0.4.8/0.4.9/0.5.0/0.5.1/0.5.2/0.5.3 +QYTISettingForiOS/0.0.1/0.0.2/0.0.3/0.0.4/0.0.5/0.0.7 +RxReusable/0.1.0/0.1.1/0.2.0/0.3.0/0.3.1 +TweetNaclClone/0.2.0 +UXAnalysis/0.0.1/0.0.4/0.0.6/0.0.7 +UrsusAtom/1.0.0/1.0.1/1.1.0/1.2.0/1.2.1/1.2.2/1.2.3/1.2.4 +WonderPush/2.0.0/2.0.1/2.1.0/2.1.1/2.1.2/2.1.3/2.2.0/2.2.1/2.3.0/3.0.0/3.1.0/3.1.1/3.1.2/3.1.3/3.1.4/3.1.5/3.1.6/3.2.0/3.3.0/3.3.1/3.3.2/3.3.3/4.0.0/4.0.1/4.0.10/4.0.11/4.0.12/4.0.2/4.0.3/4.0.4/4.0.6/4.0.9/4.1.0/4.1.1/4.1.2/4.1.3/4.1.4/4.1.5/4.1.6/4.2.0/4.2.1/4.3.0/4.3.1 +ZHViewHelper/0.0.1 +check_arch/0.1.0 diff --git a/tests/data/cocoapods/expected_construct_cocoapods_package.json b/tests/data/cocoapods/expected_construct_cocoapods_package.json new file mode 100644 index 00000000..571148c7 --- /dev/null +++ b/tests/data/cocoapods/expected_construct_cocoapods_package.json @@ -0,0 +1,36 @@ +{ + "type": "cocoapods", + "namespace": null, + "name": "KVLLibraries", + "version": "1.1.0", + "qualifiers": {}, + "subpath": null, + "repository_homepage_url": "https://cocoapods.org/pods/KVLLibraries", + "repository_download_url": null, + "api_data_url": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/KevalPatel94/KVLLibraries", + "download_url": "https://github.com/KevalPatel94/KVLLibraries/archive/refs/tags/1.1.0.tar.gz", + "api_url": "https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/5/1/f/KVLLibraries/1.1.0/KVLLibraries.podspec.json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/KevalPatel94/KVLLibraries/issues", + "code_view_url": "https://github.com/KevalPatel94/KVLLibraries", + "vcs_url": "https://github.com/KevalPatel94/KVLLibraries.git", + "copyright": null, + "license_expression": null, + "declared_license": "MIT", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:cocoapods/KVLLibraries@1.1.0" +} diff --git a/tests/data/cocoapods/get_json_response_kvllibraries.json b/tests/data/cocoapods/get_json_response_kvllibraries.json new file mode 100644 index 00000000..1ab3405c --- /dev/null +++ b/tests/data/cocoapods/get_json_response_kvllibraries.json @@ -0,0 +1,19 @@ +{ + "name": "KVLLibraries", + "version": "1.1.0", + "summary": "Library that boost IOS Development", + "description": "Super Framework for basic UIComponents, Which helps you to handle your design from storyboard", + "homepage": "https://github.com/KevalPatel94/KVLLibraries", + "license": "MIT", + "authors": { + "Keval Patel": "kevalpatelstudy@gmail.com" + }, + "platforms": { + "ios": "11.0" + }, + "source": { + "git": "https://github.com/KevalPatel94/KVLLibraries.git", + "tag": "1.1.0" + }, + "source_files": "KVLLibraries/**/*" +} diff --git a/tests/data/cocoapods/pod_summary_kvllibraries.json b/tests/data/cocoapods/pod_summary_kvllibraries.json new file mode 100644 index 00000000..3eaa3542 --- /dev/null +++ b/tests/data/cocoapods/pod_summary_kvllibraries.json @@ -0,0 +1,17 @@ +{ + "input_purl": "pkg:cocoapods/KVLLibraries", + "input_name": "KVLLibraries", + "cocoapods_org_url": "https://cocoapods.org/pods/KVLLibraries", + "repository_homepage_url": "https://cocoapods.org/pods/KVLLibraries", + "no_github_repo": null, + "gh_repo_four_o_four": "The cocoapods.org GitHub repo url for KVLLibraries returns 404", + "http_url": null, + "cocoapods_org_url_status_code": 200, + "cocoapods_org_gh_repo_url_status_code": 404, + "cocoapods_org_gh_repo_owner": "KevalPatel94", + "cocoapods_org_gh_repo_name": "KVLLibraries", + "cocoapods_org_gh_repo_url": "https://github.com/KevalPatel94/KVLLibraries", + "cocoapods_org_podspec_url": "https://github.com/CocoaPods/Specs/blob/master/Specs/5/1/f/KVLLibraries/1.1.0/KVLLibraries.podspec.json", + "cocoapods_org_pkg_home_url": null, + "cocoapods_org_pod_name": "KVLLibraries" +} diff --git a/tests/test_package.py b/tests/test_package.py index 68c93507..e4ea94dc 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -15,12 +15,20 @@ # specific language governing permissions and limitations under the License. import json +from collections import OrderedDict from unittest import TestCase from unittest import mock import pytest +from packageurl import PackageURL +from fetchcode.package import get_cocoapods_data_from_purl from fetchcode.package import info +from fetchcode.package_util import construct_cocoapods_package +from fetchcode.package_util import get_cocoapod_tags +from fetchcode.package_util import get_cocoapods_org_url_status +from fetchcode.package_util import get_pod_data_with_soup +from fetchcode.packagedcode_models import Package def file_data(file_name): @@ -29,6 +37,12 @@ def file_data(file_name): return json.loads(data) +def file_data_text(file_name): + with open(file_name) as file: + data = file.read() + return data + + def match_data(packages, expected_data): data = [dict(p.to_dict()) for p in packages] expected_data_dict = dict(expected_data) @@ -36,6 +50,12 @@ def match_data(packages, expected_data): assert expected_data == data +def match_data_list(data_list, expected_data): + data = sorted(data_list) + expected_data = sorted(expected_data) + assert expected_data == data + + @mock.patch("fetchcode.package.get_response") def test_cargo_packages(mock_get): side_effect = [file_data("tests/data/cargo_mock_data.json")] @@ -97,6 +117,188 @@ def test_tuby_package_with_invalid_url(mock_get): assert "Failed to fetch: https://rubygems.org/api/v1/gems/file.json" == e_info +# 2024-05-07 Tuesday 18:08:04. Work-in-progress. The output data leads me to believe there's still at least one live call out to the Internet. +# @mock.patch("fetchcode.package_util.construct_cocoapods_package") # variable containing result is `tag_pkg` +# @mock.patch("fetchcode.package_util.get_cocoapod_tags") # variable containing result is `data_list` +# @mock.patch("fetchcode.utils.get_hashed_path") +# @mock.patch("fetchcode.package_util.get_pod_data_with_soup") +# @mock.patch("fetchcode.package_util.get_cocoapods_org_url_status") +# def test_get_cocoapods_data_from_purl(mock_get_cocoapods_org_url_status, mock_get_pod_data_with_soup, mock_get_hashed_path, mock_get_cocoapod_tags, mock_construct_cocoapods_package): +# # def test_get_cocoapods_data_from_purl(): +# # print(f"\ntest construction in progress....") +# mock_get_cocoapods_org_url_status.return_value = {'return_message': None} + +# mock_get_pod_data_with_soup.return_value = { +# 'cocoapods_org_gh_repo_url_status_code': 200, +# 'cocoapods_org_gh_repo_owner': 'Appspia', +# 'cocoapods_org_gh_repo_name': 'ASNetworking', +# 'cocoapods_org_gh_repo_url': 'https://github.com/Appspia/ASNetworking', +# 'cocoapods_org_podspec_url': 'https://github.com/CocoaPods/Specs/blob/master/Specs/5/5/b/ASNetworking/0.1.5/ASNetworking.podspec.json', +# 'cocoapods_org_pkg_home_url': None, +# 'cocoapods_org_version': None, +# 'cocoapods_org_pod_name': 'ASNetworking', +# } + +# mock_get_hashed_path.return_value = "5/5/b" + +# mock_get_cocoapod_tags.side_effect = [ +# '0.1.5', +# '0.1.4', +# '0.1.3', +# '0.1.2', +# '0.1.1', +# '0.1.0', +# ] + +# mock_construct_cocoapods_package.side_effect = [ +# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.5'), +# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.4'), +# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.3'), +# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.2'), +# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.1'), +# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.0'), +# ] + +# expected_result = [ +# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.5'), +# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.4'), +# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.3'), +# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.2'), +# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.1'), +# Package(type='cocoapods', namespace=None, name='ASNetworking', version='0.1.0'), +# ] + +# purl = "pkg:cocoapods/ASNetworking" + +# actual_result = get_cocoapods_data_from_purl(purl) +# print(f"\nactual_result = {actual_result}") + +# # for pkg in actual_result: +# # print(pkg.to_dict()) + +# # assert list(actual_result) == expected_result + +# for pkg, expected_pkg in zip(list(actual_result), expected_result): +# assert pkg.to_dict() == expected_pkg.to_dict() + + + + + +@mock.patch("fetchcode.utils.make_head_request") +def test_get_cocoapods_org_url_status(mock_make_head_request): + mock_response = mock.Mock() + mock_response.status_code = 302 + mock_response.text = "The cocoapods.org URL https://cocoapods.org/pods/BSSimpleHTTPNetworking redirects to https://github.com/juxingzhutou/BSSimpleHTTPNetworking" + mock_response.headers = { + 'Date': 'Thu, 02 May 2024 06:02:10 GMT', + 'Content-Type': 'text/html;charset=utf-8', + 'Connection': 'keep-alive', + 'Report-To': '{"group":"heroku-nel","max_age":3600,"endpoints":[{"url":"https://nel.heroku.com/reports?ts=1714629728&sid=c46efe9b-d3d2-4a0c-8c76-bfafa16c5add&s=rPI0KHQY0J7GvkjgHpmcuMxWDuTga0k8UEFRezWRyrU%3D"}]}', + 'Reporting-Endpoints': 'heroku-nel=https://nel.heroku.com/reports?ts=1714629728&sid=c46efe9b-d3d2-4a0c-8c76-bfafa16c5add&s=rPI0KHQY0J7GvkjgHpmcuMxWDuTga0k8UEFRezWRyrU%3D', + 'Nel': '{"report_to":"heroku-nel","max_age":3600,"success_fraction":0.005,"failure_fraction":0.05,"response_headers":["Via"]}', + 'Cache-Control': 'public, max-age=20, s-maxage=60', + 'Location': 'https://github.com/juxingzhutou/BSSimpleHTTPNetworking', + 'X-Xss-Protection': '1; mode=block', + 'X-Content-Type-Options': 'nosniff', + 'X-Frame-Options': 'SAMEORIGIN', + 'Via': '1.1 vegur', + 'CF-Cache-Status': 'HIT', + 'Age': '2', + 'Vary': 'Accept-Encoding', + 'Server': 'cloudflare', + 'CF-RAY': '87d5cd05bd2cf973-SJC', + } + mock_make_head_request.return_value = mock_response + + purl = "pkg:cocoapods/BSSimpleHTTPNetworking" + name = "BSSimpleHTTPNetworking" + cocoapods_org_url = "https://cocoapods.org/pods/BSSimpleHTTPNetworking" + response = get_cocoapods_org_url_status(purl, name, cocoapods_org_url) + + assert response == { + 'corrected_name': 'BSSimpleHTTPNetworking', + 'cocoapods_org_pod_name': 'BSSimpleHTTPNetworking', + 'cocoapods_org_gh_repo_owner': 'juxingzhutou', + 'cocoapods_org_gh_repo_name': 'BSSimpleHTTPNetworking', + 'cocoapods_org_version': None, + 'return_message': 'cocoapods_org_redirects_to_github', + } + + +@mock.patch("fetchcode.utils.get_github_rest_no_exception") +@mock.patch("fetchcode.utils.make_head_request") +@mock.patch("fetchcode.utils.get_complete_response") +def test_get_pod_data_with_soup(mock_get_complete_response, mock_make_head_request, mock_get_github_rest_no_exception): + mock_complete_response = mock.MagicMock() + mock_complete_response.status_code = 200 + mock_complete_response.text = file_data_text("tests/data/cocoapods/afnetworking_response_text.txt") + mock_get_complete_response.side_effect = [mock_complete_response] + + mock_head_request_response = mock.MagicMock() + mock_head_request_response.status_code = 200 + mock_make_head_request.side_effect = [mock_head_request_response] + + mock_get_github_rest_no_exception.side_effect = [file_data("tests/data/cocoapods/afnetworking_github_rest_no_exception_response.json")] + + purl = PackageURL.from_string("pkg:cocoapods/AFNetworking@4.0.1") + name = "AFNetworking" + cocoapods_org_url = "https://cocoapods.org/pods/AFNetworking" + + soup_data = get_pod_data_with_soup(purl, name, cocoapods_org_url) + expected = { + 'cocoapods_org_gh_repo_url_status_code': 200, + 'cocoapods_org_gh_repo_owner': 'AFNetworking', + 'cocoapods_org_gh_repo_name': 'AFNetworking', + 'cocoapods_org_gh_repo_url': 'https://github.com/AFNetworking/AFNetworking', + 'cocoapods_org_podspec_url': 'https://github.com/CocoaPods/Specs/blob/master/Specs/a/7/5/AFNetworking/4.0.1/AFNetworking.podspec.json', + 'cocoapods_org_pkg_home_url': None, + 'cocoapods_org_version': None, + 'cocoapods_org_pod_name': 'AFNetworking', + } + + assert soup_data == expected + + +@mock.patch("fetchcode.utils.get_text_response") +def test_get_cocoapod_tags(mock_get): + side_effect = [file_data_text("tests/data/cocoapods/cocoapod_all_pods_versions_5_1_f.txt")] + mock_get.side_effect = side_effect + cocoapods_org_pod_name = "DeptFlow" + api = "https://cdn.cocoapods.org" + hashed_path = "5/1/f" + hashed_path_underscore = hashed_path.replace("/", "_") + file_prefix = "all_pods_versions_" + spec = f"{api}/{file_prefix}{hashed_path_underscore}.txt" + expected_data = ['0.3.0', '0.2.0', '0.1.1', '0.1.0'] + data_list = get_cocoapod_tags(spec, cocoapods_org_pod_name) + + match_data_list(data_list, expected_data) + + +@mock.patch("fetchcode.utils.get_json_response") +@mock.patch("fetchcode.utils.get_github_rest_no_exception") +def test_construct_cocoapods_package(mock_get_github_rest_no_exception, mock_get_json_response): + mock_get_github_rest_no_exception.return_value = "Failed to fetch: https://api.github.com/repos/KevalPatel94/KVLLibraries" + mock_get_json_response.return_value = file_data("tests/data/cocoapods/get_json_response_kvllibraries.json") + + expected_construct_cocoapods_package = file_data("tests/data/cocoapods/expected_construct_cocoapods_package.json") + + purl = PackageURL.from_string("pkg:cocoapods/KVLLibraries") + name = "KVLLibraries" + hashed_path = "5/1/f" + repository_homepage_url = "https://cocoapods.org/pods/KVLLibraries" + cocoapods_org_gh_repo_owner = "KevalPatel94" + cocoapods_org_gh_repo_name = "KVLLibraries" + cocoapods_org_pod_name = "KVLLibraries" + tag = "1.1.0" + actual_output = construct_cocoapods_package(purl, name, hashed_path, repository_homepage_url, cocoapods_org_gh_repo_owner, cocoapods_org_gh_repo_name, tag, cocoapods_org_pod_name) + + actual = json.dumps(actual_output.to_dict()) + expected = json.dumps(expected_construct_cocoapods_package) + assert actual == expected + + class GitHubSourceTestCase(TestCase): def check_result(self, filename, packages, regen=False): result = [p.to_dict() for p in packages]