580 lines
23 KiB
Python
580 lines
23 KiB
Python
# -*- coding: utf-8 -*-
|
|
import logging
|
|
import os
|
|
import time
|
|
import datetime
|
|
import json
|
|
import types
|
|
|
|
from requests import Session, ConnectionError, Timeout, ReadTimeout, RequestException
|
|
from simplejson import JSONDecodeError
|
|
from subzero.language import Language
|
|
|
|
from babelfish import language_converters
|
|
from subliminal import Episode, Movie
|
|
from subliminal.score import get_equivalent_release_groups
|
|
from subliminal.utils import sanitize_release_group, sanitize
|
|
from subliminal_patch.exceptions import TooManyRequests, APIThrottled
|
|
from subliminal.exceptions import DownloadLimitExceeded, AuthenticationError, ConfigurationError, ServiceUnavailable, \
|
|
ProviderError
|
|
from .mixins import ProviderRetryMixin
|
|
from subliminal_patch.subtitle import Subtitle
|
|
from subliminal.subtitle import fix_line_ending, SUBTITLE_EXTENSIONS
|
|
from subliminal_patch.providers import Provider
|
|
from subliminal_patch.subtitle import guess_matches
|
|
from subliminal_patch.utils import fix_inconsistent_naming
|
|
from subliminal.cache import region
|
|
from dogpile.cache.api import NO_VALUE
|
|
from guessit import guessit
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
SHOW_EXPIRATION_TIME = datetime.timedelta(weeks=1).total_seconds()
|
|
TOKEN_EXPIRATION_TIME = datetime.timedelta(hours=12).total_seconds()
|
|
|
|
retry_amount = 3
|
|
|
|
|
|
def fix_tv_naming(title):
|
|
"""Fix TV show titles with inconsistent naming using dictionary, but do not sanitize them.
|
|
|
|
:param str title: original title.
|
|
:return: new title.
|
|
:rtype: str
|
|
|
|
"""
|
|
return fix_inconsistent_naming(title, {"Superman & Lois": "Superman and Lois",
|
|
}, True)
|
|
|
|
|
|
def fix_movie_naming(title):
|
|
return fix_inconsistent_naming(title, {
|
|
}, True)
|
|
|
|
|
|
custom_languages = {
|
|
'pt': 'pt-PT',
|
|
'zh': 'zh-CN',
|
|
}
|
|
|
|
|
|
def to_opensubtitlescom(lang):
|
|
if lang in custom_languages.keys():
|
|
return custom_languages[lang]
|
|
else:
|
|
return lang
|
|
|
|
|
|
def from_opensubtitlescom(lang):
|
|
from_custom_languages = {v: k for k, v in custom_languages.items()}
|
|
if lang in from_custom_languages.keys():
|
|
return from_custom_languages[lang]
|
|
else:
|
|
return lang
|
|
|
|
|
|
class OpenSubtitlesComSubtitle(Subtitle):
|
|
provider_name = 'opensubtitlescom'
|
|
hash_verifiable = True
|
|
hearing_impaired_verifiable = True
|
|
|
|
def __init__(self, language, forced, hearing_impaired, page_link, file_id, releases, uploader, title, year,
|
|
hash_matched, file_hash=None, season=None, episode=None, imdb_match=False):
|
|
super(OpenSubtitlesComSubtitle, self).__init__(language, hearing_impaired, page_link)
|
|
language = Language.rebuild(language, hi=hearing_impaired, forced=forced)
|
|
|
|
self.title = title
|
|
self.year = year
|
|
self.season = season
|
|
self.episode = episode
|
|
self.releases = releases
|
|
self.release_info = releases
|
|
self.language = language
|
|
self.hearing_impaired = hearing_impaired
|
|
self.forced = forced
|
|
self.file_id = file_id
|
|
self.page_link = page_link
|
|
self.download_link = None
|
|
self.uploader = uploader
|
|
self.matches = None
|
|
self.hash = file_hash
|
|
self.encoding = 'utf-8'
|
|
self.hash_matched = hash_matched
|
|
self.imdb_match = imdb_match
|
|
|
|
@property
|
|
def id(self):
|
|
return self.file_id
|
|
|
|
def get_matches(self, video):
|
|
matches = set()
|
|
type_ = "movie" if isinstance(video, Movie) else "episode"
|
|
|
|
# handle movies and series separately
|
|
if type_ == "episode":
|
|
# series
|
|
matches.add('series')
|
|
# season
|
|
if video.season == self.season:
|
|
matches.add('season')
|
|
# episode
|
|
if video.episode == self.episode:
|
|
matches.add('episode')
|
|
# imdb
|
|
if self.imdb_match:
|
|
matches.add('series_imdb_id')
|
|
else:
|
|
# title
|
|
matches.add('title')
|
|
# imdb
|
|
if self.imdb_match:
|
|
matches.add('imdb_id')
|
|
|
|
# rest is same for both groups
|
|
|
|
# year
|
|
if video.year == self.year:
|
|
matches.add('year')
|
|
|
|
# release_group
|
|
if (video.release_group and self.releases and
|
|
any(r in sanitize_release_group(self.releases)
|
|
for r in get_equivalent_release_groups(sanitize_release_group(video.release_group)))):
|
|
matches.add('release_group')
|
|
|
|
if self.hash_matched:
|
|
matches.add('hash')
|
|
|
|
# other properties
|
|
matches |= guess_matches(video, guessit(self.releases, {"type": type_}))
|
|
|
|
self.matches = matches
|
|
|
|
return matches
|
|
|
|
|
|
class OpenSubtitlesComProvider(ProviderRetryMixin, Provider):
|
|
"""OpenSubtitlesCom Provider"""
|
|
server_url = 'https://api.opensubtitles.com/api/v1/'
|
|
|
|
languages = {Language.fromopensubtitles(lang) for lang in language_converters['szopensubtitles'].codes}
|
|
languages.update(set(Language.rebuild(lang, forced=True) for lang in languages))
|
|
languages.update(set(Language.rebuild(l, hi=True) for l in languages))
|
|
|
|
video_types = (Episode, Movie)
|
|
|
|
def __init__(self, username=None, password=None, use_hash=True, include_ai_translated=False, api_key=None):
|
|
if not all((username, password)):
|
|
raise ConfigurationError('Username and password must be specified')
|
|
|
|
if not api_key:
|
|
raise ConfigurationError('Api_key must be specified')
|
|
|
|
if not all((username, password)):
|
|
raise ConfigurationError('Username and password must be specified')
|
|
|
|
self.session = Session()
|
|
self.session.headers = {'User-Agent': os.environ.get("SZ_USER_AGENT", "Sub-Zero/2"),
|
|
'Api-Key': api_key,
|
|
'Content-Type': 'application/json'}
|
|
self.token = None
|
|
self.username = username
|
|
self.password = password
|
|
self.video = None
|
|
self.use_hash = use_hash
|
|
self.include_ai_translated = include_ai_translated
|
|
self._started = None
|
|
|
|
def initialize(self):
|
|
self._started = time.time()
|
|
|
|
if region.get("oscom_token", expiration_time=TOKEN_EXPIRATION_TIME) is NO_VALUE:
|
|
logger.debug("No cached token, we'll try to login again.")
|
|
self.login()
|
|
else:
|
|
self.token = region.get("oscom_token", expiration_time=TOKEN_EXPIRATION_TIME)
|
|
|
|
def terminate(self):
|
|
self.session.close()
|
|
|
|
def ping(self):
|
|
return self._started and (time.time() - self._started) < TOKEN_EXPIRATION_TIME
|
|
|
|
def login(self, is_retry=False):
|
|
r = self.checked(
|
|
lambda: self.session.post(self.server_url + 'login',
|
|
json={"username": self.username, "password": self.password},
|
|
allow_redirects=False,
|
|
timeout=30),
|
|
is_retry=is_retry)
|
|
|
|
try:
|
|
self.token = r.json()['token']
|
|
except (ValueError, JSONDecodeError):
|
|
log_request_response(r)
|
|
raise ProviderError("Cannot get token from provider login response")
|
|
else:
|
|
log_request_response(r, non_standard=False)
|
|
region.set("oscom_token", self.token)
|
|
|
|
@staticmethod
|
|
def sanitize_external_ids(external_id):
|
|
if isinstance(external_id, types.StringTypes):
|
|
external_id = external_id.lower().lstrip('tt').lstrip('0')
|
|
sanitized_id = external_id[:-1].lstrip('0') + external_id[-1]
|
|
return int(sanitized_id)
|
|
|
|
@region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
|
|
def search_titles(self, title):
|
|
title_id = None
|
|
|
|
parameters = {'query': title.lower()}
|
|
logging.debug('Searching using this title: %s' % title)
|
|
|
|
results = self.retry(
|
|
lambda: self.checked(
|
|
lambda: self.session.get(self.server_url + 'features', params=parameters, timeout=30),
|
|
validate_json=True,
|
|
json_key_name='data'
|
|
),
|
|
amount=retry_amount
|
|
)
|
|
|
|
# deserialize results
|
|
results_dict = results.json()['data']
|
|
|
|
# loop over results
|
|
for result in results_dict:
|
|
if 'title' in result['attributes']:
|
|
if isinstance(self.video, Episode):
|
|
if fix_tv_naming(title).lower() == result['attributes']['title'].lower() and \
|
|
(not self.video.year or self.video.year == int(result['attributes']['year'])):
|
|
title_id = result['id']
|
|
break
|
|
else:
|
|
if fix_movie_naming(title).lower() == result['attributes']['title'].lower() and \
|
|
(not self.video.year or self.video.year == int(result['attributes']['year'])):
|
|
title_id = result['id']
|
|
break
|
|
else:
|
|
continue
|
|
|
|
if title_id:
|
|
logging.debug('Found this title ID: %s' % title_id)
|
|
return self.sanitize_external_ids(title_id)
|
|
|
|
if not title_id:
|
|
logger.debug('No match found for %s' % title)
|
|
|
|
def query(self, languages, video):
|
|
self.video = video
|
|
if self.use_hash:
|
|
file_hash = self.video.hashes.get('opensubtitlescom')
|
|
logging.debug('Searching using this hash: %s' % hash)
|
|
else:
|
|
file_hash = None
|
|
|
|
if isinstance(self.video, Episode):
|
|
title = self.video.series
|
|
else:
|
|
title = self.video.title
|
|
|
|
imdb_id = None
|
|
if isinstance(self.video, Episode) and self.video.series_imdb_id:
|
|
imdb_id = self.sanitize_external_ids(self.video.series_imdb_id)
|
|
elif isinstance(self.video, Movie) and self.video.imdb_id:
|
|
imdb_id = self.sanitize_external_ids(self.video.imdb_id)
|
|
|
|
title_id = None
|
|
if not imdb_id:
|
|
title_id = self.search_titles(title)
|
|
if not title_id:
|
|
return []
|
|
|
|
# be sure to remove duplicates using list(set())
|
|
langs_list = sorted(list(set([to_opensubtitlescom(lang.basename).lower() for lang in languages])))
|
|
|
|
langs = ','.join(langs_list)
|
|
logging.debug('Searching for those languages: %s' % langs)
|
|
|
|
# query the server
|
|
if isinstance(self.video, Episode):
|
|
res = self.retry(
|
|
lambda: self.checked(
|
|
lambda: self.session.get(self.server_url + 'subtitles',
|
|
params=(('ai_translated', 'exclude' if not self.include_ai_translated
|
|
else 'include'),
|
|
('episode_number', self.video.episode),
|
|
('imdb_id', imdb_id if not title_id else None),
|
|
('languages', langs),
|
|
('moviehash', file_hash),
|
|
('parent_feature_id', title_id if title_id else None),
|
|
('season_number', self.video.season)),
|
|
timeout=30),
|
|
validate_json=True,
|
|
json_key_name='data'
|
|
),
|
|
amount=retry_amount
|
|
)
|
|
else:
|
|
res = self.retry(
|
|
lambda: self.checked(
|
|
lambda: self.session.get(self.server_url + 'subtitles',
|
|
params=(('ai_translated', 'exclude' if not self.include_ai_translated
|
|
else 'include'),
|
|
('id', title_id if title_id else None),
|
|
('imdb_id', imdb_id if not title_id else None),
|
|
('languages', langs),
|
|
('moviehash', file_hash)),
|
|
timeout=30),
|
|
validate_json=True,
|
|
json_key_name='data'
|
|
),
|
|
amount=retry_amount
|
|
)
|
|
|
|
subtitles = []
|
|
|
|
result = res.json()
|
|
|
|
# filter out forced subtitles or not depending on the required languages
|
|
if all([lang.forced for lang in languages]): # only forced
|
|
result['data'] = [x for x in result['data'] if x['attributes']['foreign_parts_only']]
|
|
elif any([lang.forced for lang in languages]): # also forced
|
|
pass
|
|
else: # not forced
|
|
result['data'] = [x for x in result['data'] if not x['attributes']['foreign_parts_only']]
|
|
|
|
logging.debug("Query returned %s subtitles" % len(result['data']))
|
|
|
|
if len(result['data']):
|
|
for item in result['data']:
|
|
# ignore AI translated subtitles
|
|
if 'ai_translated' in item['attributes'] and item['attributes']['ai_translated']:
|
|
logging.debug("Skipping AI translated subtitles")
|
|
continue
|
|
|
|
# ignore machine translated subtitles
|
|
if 'machine_translated' in item['attributes'] and item['attributes']['machine_translated']:
|
|
logging.debug("Skipping machine translated subtitles")
|
|
continue
|
|
|
|
if 'season_number' in item['attributes']['feature_details']:
|
|
season_number = item['attributes']['feature_details']['season_number']
|
|
else:
|
|
season_number = None
|
|
|
|
if 'episode_number' in item['attributes']['feature_details']:
|
|
episode_number = item['attributes']['feature_details']['episode_number']
|
|
else:
|
|
episode_number = None
|
|
|
|
if 'moviehash_match' in item['attributes']:
|
|
moviehash_match = item['attributes']['moviehash_match']
|
|
else:
|
|
moviehash_match = False
|
|
|
|
try:
|
|
year = int(item['attributes']['feature_details']['year'])
|
|
except TypeError:
|
|
year = item['attributes']['feature_details']['year']
|
|
|
|
if len(item['attributes']['files']):
|
|
subtitle = OpenSubtitlesComSubtitle(
|
|
language=Language.fromietf(from_opensubtitlescom(item['attributes']['language'])),
|
|
forced=item['attributes']['foreign_parts_only'],
|
|
hearing_impaired=item['attributes']['hearing_impaired'],
|
|
page_link=item['attributes']['url'],
|
|
file_id=item['attributes']['files'][0]['file_id'],
|
|
releases=item['attributes']['release'],
|
|
uploader=item['attributes']['uploader']['name'],
|
|
title=item['attributes']['feature_details']['movie_name'],
|
|
year=year,
|
|
season=season_number,
|
|
episode=episode_number,
|
|
hash_matched=moviehash_match,
|
|
imdb_match=True if imdb_id else False
|
|
)
|
|
subtitle.get_matches(self.video)
|
|
subtitles.append(subtitle)
|
|
|
|
return subtitles
|
|
|
|
def list_subtitles(self, video, languages):
|
|
return self.query(languages, video)
|
|
|
|
def download_subtitle(self, subtitle):
|
|
logger.info('Downloading subtitle %r', subtitle)
|
|
|
|
headers = {'Accept': 'application/json', 'Content-Type': 'application/json',
|
|
'Authorization': 'Bearer ' + self.token}
|
|
res = self.retry(
|
|
lambda: self.checked(
|
|
lambda: self.session.post(self.server_url + 'download',
|
|
json={'file_id': subtitle.file_id, 'sub_format': 'srt'},
|
|
headers=headers,
|
|
timeout=30),
|
|
validate_json=True,
|
|
json_key_name='link'
|
|
),
|
|
amount=retry_amount
|
|
)
|
|
|
|
download_data = res.json()
|
|
subtitle.download_link = download_data['link']
|
|
|
|
r = self.retry(
|
|
lambda: self.checked(
|
|
lambda: self.session.get(subtitle.download_link, timeout=30),
|
|
validate_content=True
|
|
),
|
|
amount=retry_amount
|
|
)
|
|
|
|
if not r:
|
|
logger.debug('Could not download subtitle from %s' % subtitle.download_link)
|
|
subtitle.content = None
|
|
return
|
|
else:
|
|
subtitle_content = r.content
|
|
subtitle.content = fix_line_ending(subtitle_content)
|
|
|
|
@staticmethod
|
|
def reset_token():
|
|
logging.debug('Authentication failed: clearing cache and attempting to login.')
|
|
region.delete("oscom_token")
|
|
return
|
|
|
|
def checked(self, fn, raise_api_limit=False, validate_json=False, json_key_name=None, validate_content=False,
|
|
is_retry=False):
|
|
"""Run :fn: and check the response status before returning it.
|
|
|
|
:param fn: the function to make an API call to OpenSubtitles.com.
|
|
:param raise_api_limit: if True we wait a little bit longer before running the call again.
|
|
:param validate_json: test if response is valid json.
|
|
:param json_key_name: test if returned json contain a specific key.
|
|
:param validate_content: test if response have a content (used with download).
|
|
:param is_retry: prevent additional retries with login endpoint.
|
|
:return: the response.
|
|
|
|
"""
|
|
response = None
|
|
try:
|
|
try:
|
|
response = fn()
|
|
except APIThrottled:
|
|
if not raise_api_limit:
|
|
logger.info("API request limit hit, waiting and trying again once.")
|
|
time.sleep(15)
|
|
return self.checked(fn, raise_api_limit=True)
|
|
raise
|
|
except (ConnectionError, Timeout, ReadTimeout):
|
|
raise ServiceUnavailable('Unknown Error, empty response: {}: {}'.format(response.status_code, response))
|
|
except Exception:
|
|
logging.exception('Unhandled exception raised.')
|
|
raise ProviderError('Unhandled exception raised. Check log.')
|
|
else:
|
|
status_code = response.status_code
|
|
except Exception:
|
|
status_code = None
|
|
else:
|
|
if status_code == 400:
|
|
try:
|
|
json_response = response.json()
|
|
message = json_response['message']
|
|
except JSONDecodeError:
|
|
raise ProviderError('Invalid JSON returned by provider')
|
|
else:
|
|
log_request_response(response)
|
|
raise ConfigurationError(message)
|
|
elif status_code == 401:
|
|
log_request_response(response)
|
|
self.reset_token()
|
|
if is_retry:
|
|
raise AuthenticationError('Login failed')
|
|
else:
|
|
time.sleep(1)
|
|
self.login(is_retry=True)
|
|
self.checked(fn, raise_api_limit=raise_api_limit, validate_json=validate_json,
|
|
json_key_name=json_key_name, validate_content=validate_content, is_retry=True)
|
|
elif status_code == 403:
|
|
log_request_response(response)
|
|
raise ProviderError("Bazarr API key seems to be in problem")
|
|
elif status_code == 406:
|
|
try:
|
|
json_response = response.json()
|
|
download_count = json_response['requests']
|
|
remaining_download = json_response['remaining']
|
|
quota_reset_time = json_response['reset_time']
|
|
except JSONDecodeError:
|
|
raise ProviderError('Invalid JSON returned by provider')
|
|
else:
|
|
log_request_response(response)
|
|
raise DownloadLimitExceeded("Daily download limit reached. {} subtitles have been "
|
|
"downloaded and {} remaining subtitles can be "
|
|
"downloaded. Quota will be reset in {}.".format(download_count, remaining_download, quota_reset_time))
|
|
elif status_code == 410:
|
|
log_request_response(response)
|
|
raise ProviderError("Download as expired")
|
|
elif status_code == 429:
|
|
log_request_response(response)
|
|
raise TooManyRequests()
|
|
elif status_code == 500:
|
|
logging.debug("Server side exception raised while downloading from opensubtitles.com website. They "
|
|
"should mitigate this soon.")
|
|
return None
|
|
elif status_code == 502:
|
|
# this one should deal with Bad Gateway issue on their side.
|
|
raise APIThrottled()
|
|
elif 500 <= status_code <= 599:
|
|
raise ProviderError(response.reason)
|
|
|
|
if status_code != 200:
|
|
log_request_response(response)
|
|
raise ProviderError('Bad status code: %s' % response.status_code)
|
|
|
|
if validate_json:
|
|
try:
|
|
json_test = response.json()
|
|
except JSONDecodeError:
|
|
raise ProviderError('Invalid JSON returned by provider')
|
|
else:
|
|
if json_key_name not in json_test:
|
|
raise ProviderError('Invalid JSON returned by provider: no %s key in returned json.' % json_key_name)
|
|
|
|
if validate_content:
|
|
if not hasattr(response, 'content'):
|
|
logging.error('Download link returned no content attribute.')
|
|
return False
|
|
elif not response.content:
|
|
logging.error('This download link returned empty content: %s' % response.url)
|
|
return False
|
|
|
|
return response
|
|
|
|
|
|
def log_request_response(response, non_standard=True):
|
|
redacted_request_headers = response.request.headers
|
|
if 'Authorization' in redacted_request_headers and isinstance(redacted_request_headers['Authorization'], str):
|
|
redacted_request_headers['Authorization'] = redacted_request_headers['Authorization'][:-8]+8*'x'
|
|
|
|
redacted_request_body = json.loads(response.request.body)
|
|
if 'password' in redacted_request_body:
|
|
redacted_request_body['password'] = 'redacted'
|
|
|
|
redacted_response_body = json.loads(response.text)
|
|
if 'token' in redacted_response_body and isinstance(redacted_response_body['token'], str):
|
|
redacted_response_body['token'] = redacted_response_body['token'][:-8] + 8 * 'x'
|
|
|
|
if non_standard:
|
|
logging.debug("opensubtitlescom returned a non standard response. Logging request/response for debugging "
|
|
"purpose.")
|
|
else:
|
|
logging.debug("opensubtitlescom returned a standard response. Logging request/response for debugging purpose.")
|
|
logging.debug("Request URL: %s" % response.request.url)
|
|
logging.debug("Request Headers: %s" % redacted_request_headers)
|
|
logging.debug("Request Body: %s" % json.dumps(redacted_request_body))
|
|
logging.debug("Response Status Code: %s" % {response.status_code})
|
|
logging.debug("Response Headers: %s" % response.headers)
|
|
logging.debug("Response Body: %s" % json.dumps(redacted_response_body))
|