Add cocoapods support to package.py

requirements.txt

@@ -73,6 +73,7 @@ text-unidecode==1.3
 toml==0.10.2
 typecode==30.0.0
 typecode-libmagic==5.39.210531
+univers==30.11.0
 urllib3==1.26.9
 urlpy==0.5
 wcwidth==0.2.5

setup.cfg

@@ -59,6 +59,7 @@ install_requires =
     requests
     python-dateutil
     python-dotenv
+    univers >= 30.11.0
 
 
 [options.packages.find]
@@ -80,4 +81,3 @@ docs =
     sphinx-rtd-theme>=1.0.0
     sphinx-reredirects >= 0.1.2
     doc8>=0.11.2
-

src/fetchcode/package.py

@@ -32,7 +32,10 @@
 from fetchcode.package_util import GitHubSource
 from fetchcode.package_util import MiniupnpPackagesGitHubSource
 from fetchcode.package_util import OpenSSLGitHubSource
+from fetchcode.package_util import construct_cocoapods_package
+from fetchcode.package_util import get_cocoapod_tags
 from fetchcode.packagedcode_models import Package
+from fetchcode.utils import get_hashed_path
 from fetchcode.utils import get_response
 
 router = Router()
@@ -362,6 +365,50 @@ def get_gnu_data_from_purl(purl):
     )
 
 
+@router.route("pkg:cocoapods/.*")
+def get_cocoapods_data_from_purl(purl):
+    purl = PackageURL.from_string(purl)
+    name = purl.name
+    cocoapods_org_url = f"https://cocoapods.org/pods/{name}"
+    api = "https://cdn.cocoapods.org"
+    hashed_path = get_hashed_path(name)
+    hashed_path_underscore = hashed_path.replace("/", "_")
+    file_prefix = "all_pods_versions_"
+    spec = f"{api}/{file_prefix}{hashed_path_underscore}.txt"
+    data_list = get_cocoapod_tags(spec, name)
+
+    for tag in data_list:
+        if purl.version and tag != purl.version:
+            continue
+
+        gh_repo_owner = None
+        gh_repo_name = name
+        podspec_api_url = f"https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/{hashed_path}/{name}/{tag}/{name}.podspec.json"
+        podspec_api_response = get_response(podspec_api_url)
+        podspec_homepage = podspec_api_response.get('homepage')
+
+        if podspec_homepage.startswith("https://github.com/"):
+            podspec_homepage_remove_gh_prefix = podspec_homepage.replace("https://github.com/", "")
+            podspec_homepage_split = podspec_homepage_remove_gh_prefix.split("/")
+            gh_repo_owner = podspec_homepage_split[0]
+            gh_repo_name = podspec_homepage_split[-1]
+
+        tag_pkg = construct_cocoapods_package(
+            purl,
+            name,
+            hashed_path,
+            cocoapods_org_url,
+            gh_repo_owner,
+            gh_repo_name,
+            tag
+        )
+
+        yield tag_pkg
+
+        if purl.version:
+            break
+
+
 @dataclasses.dataclass
 class DirectoryListedSource:
     source_url: str = dataclasses.field(

src/fetchcode/package_util.py

@@ -723,3 +723,107 @@ def get_package_info(cls, gh_purl, package_name):
         "date": "2002-08-19T04:23:00",
     },
 }
+
+
+def get_cocoapod_tags(spec, name):
+    try:
+        response = utils.get_text_response(spec)
+        data = response.strip()
+        for line in data.splitlines():
+            line = line.strip()
+            if line.startswith(name):
+                data_list = line.split("/")
+                if data_list[0] == name:
+                    data_list.pop(0)
+                return data_list
+        return None
+    except:
+        return None
+
+
+def construct_cocoapods_package(
+    purl,
+    name,
+    hashed_path,
+    cocoapods_org_url,
+    gh_repo_owner,
+    gh_repo_name,
+    tag
+):
+    name = name
+    homepage_url = None
+    vcs_url = None
+    github_url = None
+    bug_tracking_url = None
+    code_view_url = None
+    license_data = None
+    declared_license = None
+    primary_language = None
+
+    if gh_repo_owner and gh_repo_name:
+        base_path = "https://api.github.com/repos"
+        api_url = f"{base_path}/{gh_repo_owner}/{gh_repo_name}"
+        gh_repo_api_response = utils.get_github_rest(api_url)
+        gh_repo_api_head_request = utils.make_head_request(api_url)
+        gh_repo_api_status_code = gh_repo_api_head_request.status_code
+
+        if gh_repo_api_status_code == 200:
+            homepage_url = gh_repo_api_response.get("homepage")
+            vcs_url = gh_repo_api_response.get("git_url")
+            license_data = gh_repo_api_response.get("license") or {}
+            declared_license = license_data.get("spdx_id")
+            primary_language = gh_repo_api_response.get("language")
+
+        github_url = "https://github.com"
+        bug_tracking_url = f"{github_url}/{gh_repo_owner}/{gh_repo_name}/issues"
+        code_view_url = f"{github_url}/{gh_repo_owner}/{gh_repo_name}"
+
+    podspec_api_url = f"https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/{hashed_path}/{name}/{tag}/{name}.podspec.json"
+    podspec_api_response = utils.get_response(podspec_api_url)
+    homepage_url = podspec_api_response.get("homepage")
+
+    lic = podspec_api_response.get("license")
+    extracted_license_statement = None
+    if isinstance(lic, dict):
+        extracted_license_statement = lic
+    else:
+        extracted_license_statement = lic
+    if not declared_license:
+        declared_license = extracted_license_statement
+
+    source = podspec_api_response.get("source")
+    download_url = None
+    if isinstance(source, dict):
+        git_url = source.get("git", "")
+        http_url = source.get("http", "")
+        if http_url:
+            download_url = http_url
+        if git_url and not http_url:
+            if git_url.endswith(".git") and git_url.startswith("https://github.com/"):
+                gh_path = git_url[:-4]
+                github_tag = source.get("tag")
+                if github_tag and github_tag.startswith("v"):
+                    tag = github_tag
+                download_url = f"{gh_path}/archive/refs/tags/{tag}.tar.gz"
+                vcs_url = git_url
+        elif git_url:
+            vcs_url = git_url
+    elif isinstance(source, str):
+        if not vcs_url:
+            vcs_url = source
+
+    purl_pkg = Package(
+        homepage_url=homepage_url,
+        api_url=podspec_api_url,
+        bug_tracking_url=bug_tracking_url,
+        code_view_url=code_view_url,
+        download_url=download_url,
+        declared_license=declared_license,
+        primary_language=primary_language,
+        repository_homepage_url=cocoapods_org_url,
+        vcs_url=vcs_url,
+        **purl.to_dict(),
+    )
+    purl_pkg.version = tag
+
+    return purl_pkg

src/fetchcode/packagedcode_models.py

@@ -27,23 +27,21 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from collections import OrderedDict
 import logging
 import sys
+from collections import OrderedDict
 
 import attr
-from packageurl import normalize_qualifiers
-from packageurl import PackageURL
-
-from commoncode.datautils import choices
 from commoncode.datautils import Boolean
 from commoncode.datautils import Date
 from commoncode.datautils import Integer
 from commoncode.datautils import List
 from commoncode.datautils import Mapping
 from commoncode.datautils import String
 from commoncode.datautils import TriBoolean
-
+from commoncode.datautils import choices
+from packageurl import PackageURL
+from packageurl import normalize_qualifiers
 
 """
 Data models for package information and dependencies, abstracting the
@@ -247,31 +245,24 @@ def purl(self):
             self.type, self.namespace, self.name, self.version,
             self.qualifiers, self.subpath).to_string()
 
-    def repository_homepage_url(self, baseurl=default_web_baseurl):
-        """
-        Return the package repository homepage URL for this package, e.g. the
-        URL to the page for this package in its package repository. This is
-        typically different from the package homepage URL proper.
-        Subclasses should override to provide a proper value.
-        """
-        return
-
-    def repository_download_url(self, baseurl=default_download_baseurl):
-        """
-        Return the package repository download URL to download the actual
-        archive of code of this package. This may be different than the actual
-        download URL and is computed from the default public respoitory baseurl.
-        Subclasses should override to provide a proper value.
-        """
-        return
-
-    def api_data_url(self, baseurl=default_api_baseurl):
-        """
-        Return the package repository API URL to obtain structured data for this
-        package such as the URL to a JSON or XML api.
-        Subclasses should override to provide a proper value.
-        """
-        return
+    repository_homepage_url = String(
+        label='package repository homepage URL.',
+        help='URL to the page for this package in its package repository. '
+             'This is typically different from the package homepage URL proper.'
+     )
+
+    repository_download_url = String(
+        label='package repository download URL.',
+        help='download URL to download the actual archive of code of this '
+             'package in its package repository. '
+             'This may be different from the actual download URL.'
+     )
+
+    api_data_url = String(
+        label='package repository API URL.',
+        help='API URL to obtain structured data for this package such as the '
+             'URL to a JSON or XML api its package repository.'
+     )
 
     def set_purl(self, package_url):
         """
@@ -298,9 +289,9 @@ def to_dict(self, **kwargs):
         mapping = attr.asdict(self, dict_factory=OrderedDict)
         if not kwargs.get('exclude_properties'):
             mapping['purl'] = self.purl
-            mapping['repository_homepage_url'] = self.repository_homepage_url()
-            mapping['repository_download_url'] = self.repository_download_url()
-            mapping['api_data_url'] = self.api_data_url()
+            mapping['repository_homepage_url'] = self.repository_homepage_url
+            mapping['repository_download_url'] = self.repository_download_url
+            mapping['api_data_url'] = self.api_data_url
         if self.qualifiers:
             mapping['qualifiers'] = normalize_qualifiers(self.qualifiers, encode=False)
         return mapping
@@ -399,7 +390,7 @@ class Package(BasePackage):
 
     api_url = String(
         label='API URL',
-        help='URL of API for this package.')    
+        help='URL of API for this package.')
 
     size = Integer(
         default=None,
@@ -842,4 +833,4 @@ class SquashfsPackage(Package):
     default_type = 'squashfs'
 
 
-# TODO: Add VM images formats(VMDK, OVA, OVF, VDI, etc) and Docker/other containers
+# TODO: Add VM images formats(VMDK, OVA, OVF, VDI, etc) and Docker/other containers

src/fetchcode/utils.py

@@ -14,7 +14,11 @@
 # CONDITIONS OF ANY KIND, either express or implied. See tshe License for the
 # specific language governing permissions and limitations under the License.
 
+import hashlib
 import os
+import sys
+from functools import partial
+
 import requests
 from dateutil import parser as dateparser
 from dateutil.parser import ParserError
@@ -166,11 +170,76 @@ def get_github_rest(url):
 
 
 def get_response(url, headers=None):
-    """
-    Generate `Package` object for a `url` string
-    """
     resp = requests.get(url, headers=headers)
     if resp.status_code == 200:
         return resp.json()
 
     raise Exception(f"Failed to fetch: {url}")
+
+
+def get_text_response(url, headers=None):
+    resp = requests.get(url, headers=headers)
+    if resp.status_code == 200:
+        return resp.text
+
+    raise Exception(f"Failed to fetch: {url}")
+
+
+def make_head_request(url, headers=None):
+    try:
+        resp = requests.head(url, headers=headers)
+        return resp
+    except:
+        raise Exception(f"Failed to fetch: {url}")
+
+
+def get_hashed_path(name):
+    """
+    Returns a string with a part of the file path derived from the md5 hash.
+
+    From https://github.com/CocoaPods/cdn.cocoapods.org:
+        "There are a set of known prefixes for all Podspec paths, you take the
+        name of the pod, create a hash (using md5) of it and take the first
+        three characters."
+
+    """
+    if not name:
+        return
+    podname = get_podname_proper(name)
+    if name != podname:
+        name_to_hash = podname
+    else:
+        name_to_hash = name
+
+    hash_init = get_first_three_md5_hash_characters(name_to_hash)
+    hashed_path = "/".join(list(hash_init))
+
+    return hashed_path
+
+
+# for FIPS support
+sys_v0 = sys.version_info[0]
+sys_v1 = sys.version_info[1]
+if sys_v0 == 3 and sys_v1 >= 9:
+    md5_hasher = partial(hashlib.md5, usedforsecurity=False)
+else:
+    md5_hasher = hashlib.md5
+
+
+def get_podname_proper(podname):
+    """
+    Podnames in cocoapods sometimes are files inside a pods package (like 'OHHTTPStubs/Default')
+    This returns proper podname in those cases.
+    """
+    if "/" in podname:
+        return podname.split("/")[0]
+    return podname
+
+
+def get_first_three_md5_hash_characters(podname):
+    """
+    From https://github.com/CocoaPods/cdn.cocoapods.org:
+    "There are a set of known prefixes for all Podspec paths, you take the name of the pod,
+    create a hash (using md5) of it and take the first three characters."
+    """
+    return md5_hasher(podname.encode("utf-8")).hexdigest()[0:3]