Merge pull request #538 from rmrector/metadata.themoviedb.org.python@…

…nexus

[metadata.themoviedb.org.python@nexus] 3.0.0

metadata.themoviedb.org.python/README.md

@@ -0,0 +1,16 @@
+## The Movie Database Python scraper for Kodi
+
+This is early work on a Python movie scraper for Kodi.
+
+### Manual search by IMDB / TMDB ID
+When manually searching you can enter an IMDB or TMDB ID to pull up an exact movie result.
+To search by TMDB enter "tmdb/" then the ID, like "tmdb/11". To search by IMDB ID enter it directly.
+
+## Development info
+
+### How to run unit tests
+
+`python3 -m unittest discover -v` from the main **metadata.themoviedb.org.python** directory.
+
+Set env variable `TEST_E2E` to enable the single IMDB end-to-end test, `TEST_E2E=true python3 -m unittest discover -v`.
+Not for a pipeline, but may be helpful to run now and then.

metadata.themoviedb.org.python/changelog.txt

@@ -0,0 +1,93 @@
+v3.0.0 (2024-04-17)
+- version 3 for Kodi 20 Nexus and above
+
+v2.2.1 (2024-04-12)
+- Update YouTube plugin URL for trailers
+
+v2.2.0 (2024-01-10)
+- Support IMDB/TMDB IDs in filename for Kodi 21 Omega (uniqueIDs directly from Kodi)
+
+v2.1.0 (2023-03-18)
+- Add option to disable posters
+- Fix fallback language for landscape images
+- Don't add SVG image artwork
+
+v2.0.0 (2023-01-01)
+- version 2 for Kodi 19 Matrix and above
+- TMDB search second page if no best match on first page
+
+v1.6.2 (2022-04-10)
+- Fix: IMDB ratings
+
+v1.6.1 (2022-01-09)
+- Feature: fanart prioritization option
+
+v1.6.0 (2021-12-24)
+- Feature: fallback to English language for Fanart.tv artwork
+
+v1.5.1 (2021-10-19)
+- Fix: search error when no year
+
+v1.5.0 (2021-10-16)
+- Feature: downloading logos from tmdb
+- Feature: search language option added
+- Change: searching movies from different years if not found
+- Fix: don't error when all fanart disabled
+- Fix: don't try to fetch movie set artwork from Fanart.TV if movie is not part of set
+- skip IMDB rating tests
+
+v1.4.0 (2021-07-10)
+- Feature: update to new IMDB page layout
+- Feature: update language files and translation system
+
+v1.3.3 (2021-05-16)
+- Fix: fix collection image fallback from TMDB
+
+v1.3.2 (2021-03-13)
+- Change: improve best match selection
+- Change: poster language fallback to highest rated
+- Fix: handle errors when connecting to TMDB
+
+v1.3.1 (2020-11-02)
+- Change: simplify artwork selection options
+- Fix: strip region to pick correct poster language
+
+v1.3.0 (2020-10-04)
+- Change: removed dependencies on requests, tmdbsimple, and trakt modules
+- Change: images now returned with initial API call instead of during fallback
+- Change: settings language for TMDb now use culture name (i.e. en-US) - required for direct API call
+
+v1.2.1 (2020-08-08)
+- Fix: Prefer movies that exactly match search title and year
+- Fix: Change 'landscape from TMDb' option disabled behavior to keep titled fanart
+- Fix: Don't dupe Writers if listed with multiple jobs
+- Fix: Capitalize country code in all language options
+
+v1.2.0 (2020-05-25)
+- Feature: add extended artwork from Fanart.tv
+- Feature: separate 'fanart' images with language to 'landscape' art type
+
+v1.1.1 (2020-03-01)
+- Fix: release fixup
+
+v1.1.0 (2020-02-26)
+- Feature: option to add plot keywords from TMDB as tags
+
+v1.0.0 (2020-01-26)
+- Feature: option to enable/disable IMDB and Trakt ratings
+
+v0.7.0 (2020-01-11) - release candidate
+- Feature: add trakt rating
+- Feature: search by IMDB or TMDB ID
+- Fix: support path-specific settings
+
+v0.6.0 (2019-07-04)
+- Feature: add setting to configure certification prefix
+- Feature: add option to return single or multiple studios
+- Feature: add movie set overview and artwork
+- Fix: IMDB top 250 and ratings
+- Fix: parsing NFO file for URL / ID
+
+v0.5.0 (2019-06-09)
+- first Python version
+- early version mostly by @phate89, with an old version of tmdbsimple

metadata.themoviedb.org.python/python/lib/tmdbscraper/__init__.py

@@ -0,0 +1,17 @@
+
+def get_imdb_id(uniqueids):
+    imdb_id = uniqueids.get('imdb')
+    if not imdb_id or not imdb_id.startswith('tt'):
+        return None
+    return imdb_id
+
+# example format for scraper results
+_ScraperResults = {
+    'info',
+    'ratings',
+    'uniqueids',
+    'cast',
+    'available_art',
+    'error',
+    'warning' # not handled
+}

metadata.themoviedb.org.python/python/lib/tmdbscraper/api_utils.py

@@ -0,0 +1,85 @@
+# coding: utf-8
+#
+# Copyright (C) 2020, Team Kodi
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Functions to interact with various web site APIs."""
+
+from __future__ import absolute_import, unicode_literals
+
+import json
+
+try:
+    import xbmc
+except ModuleNotFoundError:
+    # only used for logging HTTP calls, not available nor needed for testing
+    xbmc = None
+
+# from pprint import pformat
+try: #PY2 / PY3
+    from urllib2 import Request, urlopen
+    from urllib2 import URLError
+    from urllib import urlencode
+except ImportError:
+    from urllib.request import Request, urlopen
+    from urllib.error import URLError
+    from urllib.parse import urlencode
+try:
+    from typing import Text, Optional, Union, List, Dict, Any  # pylint: disable=unused-import
+    InfoType = Dict[Text, Any]  # pylint: disable=invalid-name
+except ImportError:
+    pass
+
+HEADERS = {}
+
+
+def set_headers(headers):
+    HEADERS.update(headers)
+
+
+def load_info(url, params=None, default=None, resp_type = 'json'):
+    # type: (Text, Optional[Dict[Text, Union[Text, List[Text]]]]) -> Union[dict, list]
+    """
+    Load info from external api
+
+    :param url: API endpoint URL
+    :param params: URL query params
+    :default: object to return if there is an error
+    :resp_type: what to return to the calling function
+    :return: API response or default on error
+    """
+    theerror = ''
+    if params:
+        url = url + '?' + urlencode(params)
+    if xbmc:
+        xbmc.log('Calling URL "{}"'.format(url), xbmc.LOGDEBUG)
+    req = Request(url, headers=HEADERS)
+    try:
+        response = urlopen(req)
+    except URLError as e:
+        if hasattr(e, 'reason'):
+            theerror = {'error': 'failed to reach the remote site\nReason: {}'.format(e.reason)}
+        elif hasattr(e, 'code'):
+            theerror = {'error': 'remote site unable to fulfill the request\nError code: {}'.format(e.code)}
+        if default is not None:
+            return default
+        else:
+            return theerror
+    if resp_type.lower() == 'json':
+        resp = json.loads(response.read().decode('utf-8'))
+    else:
+        resp = response.read().decode('utf-8')
+    # xbmc.log('the api response:\n{}'.format(pformat(resp)), xbmc.LOGDEBUG)
+    return resp

metadata.themoviedb.org.python/python/lib/tmdbscraper/fanarttv.py

@@ -0,0 +1,87 @@
+from . import api_utils
+try:
+    from urllib import quote
+except ImportError: # py2 / py3
+    from urllib.parse import quote
+
+API_KEY = '384afe262ee0962545a752ff340e3ce4'
+API_URL = 'https://webservice.fanart.tv/v3/movies/{}'
+
+ARTMAP = {
+    'movielogo': 'clearlogo',
+    'hdmovielogo': 'clearlogo',
+    'hdmovieclearart': 'clearart',
+    'movieart': 'clearart',
+    'moviedisc': 'discart',
+    'moviebanner': 'banner',
+    'moviethumb': 'landscape',
+    'moviebackground': 'fanart',
+    'movieposter': 'poster'
+}
+
+def get_details(uniqueids, clientkey, language, set_tmdbid):
+    media_id = _get_mediaid(uniqueids)
+    if not media_id:
+        return {}
+
+    movie_data = _get_data(media_id, clientkey)
+    movieset_data = _get_data(set_tmdbid, clientkey) if set_tmdbid else None
+    if not movie_data and not movieset_data:
+        return {}
+
+    movie_art = {}
+    movieset_art = {}
+    if movie_data:
+        movie_art = _parse_data(movie_data, language)
+    if movieset_data:
+        movieset_art = _parse_data(movieset_data, language)
+        movieset_art = {'set.' + key: value for key, value in movieset_art.items()}
+
+    available_art = movie_art
+    available_art.update(movieset_art)
+
+    return {'available_art': available_art}
+
+def _get_mediaid(uniqueids):
+    for source in ('tmdb', 'imdb', 'unknown'):
+        if source in uniqueids:
+            return uniqueids[source]
+
+def _get_data(media_id, clientkey):
+    headers = {'api-key': API_KEY}
+    if clientkey:
+        headers['client-key'] = clientkey
+    api_utils.set_headers(headers)
+    fanarttv_url = API_URL.format(media_id)
+    return api_utils.load_info(fanarttv_url, default={})
+
+def _parse_data(data, language, language_fallback='en'):
+    result = {}
+    for arttype, artlist in data.items():
+        if arttype not in ARTMAP:
+            continue
+        for image in artlist:
+            image_lang = _get_imagelanguage(arttype, image)
+            if image_lang and image_lang != language and image_lang != language_fallback:
+                continue
+
+            generaltype = ARTMAP[arttype]
+            if generaltype == 'poster' and not image_lang:
+                generaltype = 'keyart'
+            if artlist and generaltype not in result:
+                result[generaltype] = []
+
+            url = quote(image['url'], safe="%/:=&?~#+!$,;'@()*[]")
+            resultimage = {'url': url, 'preview': url.replace('.fanart.tv/fanart/', '.fanart.tv/preview/'), 'lang': image_lang}
+            result[generaltype].append(resultimage)
+
+    return result
+
+def _get_imagelanguage(arttype, image):
+    if 'lang' not in image or arttype == 'moviebackground':
+        return None
+    if arttype in ('movielogo', 'hdmovielogo', 'hdmovieclearart', 'movieart', 'moviebanner',
+            'moviethumb', 'moviedisc'):
+        return image['lang'] if image['lang'] not in ('', '00') else 'en'
+    # movieposter may or may not have a title and thus need a language
+    return image['lang'] if image['lang'] not in ('', '00') else None