mirror of
https://github.com/rommapp/romm.git
synced 2026-04-23 06:54:40 +00:00
mega ton of fixes for 4.8
This commit is contained in:
+1
-1
@@ -58,7 +58,7 @@ RUN npm install
|
||||
WORKDIR /app
|
||||
|
||||
# Install uv for the non-root user
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.7.19 /uv /uvx /usr/local/bin/
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.2 /uv /uvx /usr/local/bin/
|
||||
|
||||
# Install Python
|
||||
RUN uv python install 3.13
|
||||
|
||||
@@ -45,6 +45,7 @@ DEFAULT_EXCLUDED_FILES: Final = [
|
||||
".stfolder",
|
||||
"@SynoResource",
|
||||
"gamelist.xml",
|
||||
"metadata.pegasus.xml",
|
||||
]
|
||||
DEFAULT_EXCLUDED_DIRS: Final = [
|
||||
"@eaDir",
|
||||
@@ -100,12 +101,12 @@ class NetplayICEServer(TypedDict):
|
||||
class Config:
|
||||
CONFIG_FILE_MOUNTED: bool
|
||||
CONFIG_FILE_WRITABLE: bool
|
||||
EXCLUDED_PLATFORMS: list[str]
|
||||
EXCLUDED_SINGLE_EXT: list[str]
|
||||
EXCLUDED_SINGLE_FILES: list[str]
|
||||
EXCLUDED_MULTI_FILES: list[str]
|
||||
EXCLUDED_MULTI_PARTS_EXT: list[str]
|
||||
EXCLUDED_MULTI_PARTS_FILES: list[str]
|
||||
EXCLUDED_PLATFORMS: set[str]
|
||||
EXCLUDED_SINGLE_EXT: set[str]
|
||||
EXCLUDED_SINGLE_FILES: set[str]
|
||||
EXCLUDED_MULTI_FILES: set[str]
|
||||
EXCLUDED_MULTI_PARTS_EXT: set[str]
|
||||
EXCLUDED_MULTI_PARTS_FILES: set[str]
|
||||
GAMELIST_AUTO_EXPORT_ON_SCAN: bool
|
||||
PLATFORMS_BINDING: dict[str, str]
|
||||
PLATFORMS_VERSIONS: dict[str, str]
|
||||
@@ -224,40 +225,56 @@ class ConfigManager:
|
||||
self.config = Config(
|
||||
CONFIG_FILE_MOUNTED=self._config_file_mounted,
|
||||
CONFIG_FILE_WRITABLE=self._config_file_writable,
|
||||
EXCLUDED_PLATFORMS=pydash.get(
|
||||
self._raw_config, "exclude.platforms", DEFAULT_EXCLUDED_DIRS
|
||||
),
|
||||
EXCLUDED_SINGLE_EXT=[
|
||||
e.lower()
|
||||
for e in pydash.get(
|
||||
EXCLUDED_PLATFORMS={
|
||||
*DEFAULT_EXCLUDED_DIRS,
|
||||
*pydash.get(self._raw_config, "exclude.platforms", []),
|
||||
},
|
||||
EXCLUDED_SINGLE_EXT={
|
||||
*(e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS),
|
||||
*(
|
||||
e.lower()
|
||||
for e in pydash.get(
|
||||
self._raw_config,
|
||||
"exclude.roms.single_file.extensions",
|
||||
[],
|
||||
)
|
||||
),
|
||||
},
|
||||
EXCLUDED_SINGLE_FILES={
|
||||
*DEFAULT_EXCLUDED_FILES,
|
||||
*pydash.get(
|
||||
self._raw_config,
|
||||
"exclude.roms.single_file.extensions",
|
||||
DEFAULT_EXCLUDED_EXTENSIONS,
|
||||
)
|
||||
],
|
||||
EXCLUDED_SINGLE_FILES=pydash.get(
|
||||
self._raw_config,
|
||||
"exclude.roms.single_file.names",
|
||||
DEFAULT_EXCLUDED_FILES,
|
||||
),
|
||||
EXCLUDED_MULTI_FILES=pydash.get(
|
||||
self._raw_config,
|
||||
"exclude.roms.multi_file.names",
|
||||
DEFAULT_EXCLUDED_DIRS,
|
||||
),
|
||||
EXCLUDED_MULTI_PARTS_EXT=[
|
||||
e.lower()
|
||||
for e in pydash.get(
|
||||
"exclude.roms.single_file.names",
|
||||
[],
|
||||
),
|
||||
},
|
||||
EXCLUDED_MULTI_FILES={
|
||||
*DEFAULT_EXCLUDED_DIRS,
|
||||
*pydash.get(
|
||||
self._raw_config,
|
||||
"exclude.roms.multi_file.parts.extensions",
|
||||
DEFAULT_EXCLUDED_EXTENSIONS,
|
||||
)
|
||||
],
|
||||
EXCLUDED_MULTI_PARTS_FILES=pydash.get(
|
||||
self._raw_config,
|
||||
"exclude.roms.multi_file.parts.names",
|
||||
DEFAULT_EXCLUDED_FILES,
|
||||
),
|
||||
"exclude.roms.multi_file.names",
|
||||
[],
|
||||
),
|
||||
},
|
||||
EXCLUDED_MULTI_PARTS_EXT={
|
||||
*(e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS),
|
||||
*(
|
||||
e.lower()
|
||||
for e in pydash.get(
|
||||
self._raw_config,
|
||||
"exclude.roms.multi_file.parts.extensions",
|
||||
[],
|
||||
)
|
||||
),
|
||||
},
|
||||
EXCLUDED_MULTI_PARTS_FILES={
|
||||
*DEFAULT_EXCLUDED_FILES,
|
||||
*pydash.get(
|
||||
self._raw_config,
|
||||
"exclude.roms.multi_file.parts.names",
|
||||
[],
|
||||
),
|
||||
},
|
||||
PLATFORMS_BINDING=pydash.get(self._raw_config, "system.platforms", {}),
|
||||
PLATFORMS_VERSIONS=pydash.get(self._raw_config, "system.versions", {}),
|
||||
ROMS_FOLDER_NAME=pydash.get(
|
||||
@@ -377,35 +394,35 @@ class ConfigManager:
|
||||
|
||||
def _validate_config(self):
|
||||
"""Validates the config.yml file"""
|
||||
if not isinstance(self.config.EXCLUDED_PLATFORMS, list):
|
||||
if not isinstance(self.config.EXCLUDED_PLATFORMS, set):
|
||||
log.critical("Invalid config.yml: exclude.platforms must be a list")
|
||||
sys.exit(3)
|
||||
|
||||
if not isinstance(self.config.EXCLUDED_SINGLE_EXT, list):
|
||||
if not isinstance(self.config.EXCLUDED_SINGLE_EXT, set):
|
||||
log.critical(
|
||||
"Invalid config.yml: exclude.roms.single_file.extensions must be a list"
|
||||
)
|
||||
sys.exit(3)
|
||||
|
||||
if not isinstance(self.config.EXCLUDED_SINGLE_FILES, list):
|
||||
if not isinstance(self.config.EXCLUDED_SINGLE_FILES, set):
|
||||
log.critical(
|
||||
"Invalid config.yml: exclude.roms.single_file.names must be a list"
|
||||
)
|
||||
sys.exit(3)
|
||||
|
||||
if not isinstance(self.config.EXCLUDED_MULTI_FILES, list):
|
||||
if not isinstance(self.config.EXCLUDED_MULTI_FILES, set):
|
||||
log.critical(
|
||||
"Invalid config.yml: exclude.roms.multi_file.names must be a list"
|
||||
)
|
||||
sys.exit(3)
|
||||
|
||||
if not isinstance(self.config.EXCLUDED_MULTI_PARTS_EXT, list):
|
||||
if not isinstance(self.config.EXCLUDED_MULTI_PARTS_EXT, set):
|
||||
log.critical(
|
||||
"Invalid config.yml: exclude.roms.multi_file.parts.extensions must be a list"
|
||||
)
|
||||
sys.exit(3)
|
||||
|
||||
if not isinstance(self.config.EXCLUDED_MULTI_PARTS_FILES, list):
|
||||
if not isinstance(self.config.EXCLUDED_MULTI_PARTS_FILES, set):
|
||||
log.critical(
|
||||
"Invalid config.yml: exclude.roms.multi_file.parts.names must be a list"
|
||||
)
|
||||
@@ -576,17 +593,17 @@ class ConfigManager:
|
||||
|
||||
self._raw_config = {
|
||||
"exclude": {
|
||||
"platforms": self.config.EXCLUDED_PLATFORMS,
|
||||
"platforms": sorted(self.config.EXCLUDED_PLATFORMS),
|
||||
"roms": {
|
||||
"single_file": {
|
||||
"extensions": self.config.EXCLUDED_SINGLE_EXT,
|
||||
"names": self.config.EXCLUDED_SINGLE_FILES,
|
||||
"extensions": sorted(self.config.EXCLUDED_SINGLE_EXT),
|
||||
"names": sorted(self.config.EXCLUDED_SINGLE_FILES),
|
||||
},
|
||||
"multi_file": {
|
||||
"names": self.config.EXCLUDED_MULTI_FILES,
|
||||
"names": sorted(self.config.EXCLUDED_MULTI_FILES),
|
||||
"parts": {
|
||||
"extensions": self.config.EXCLUDED_MULTI_PARTS_EXT,
|
||||
"names": self.config.EXCLUDED_MULTI_PARTS_FILES,
|
||||
"extensions": sorted(self.config.EXCLUDED_MULTI_PARTS_EXT),
|
||||
"names": sorted(self.config.EXCLUDED_MULTI_PARTS_FILES),
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -37,12 +37,12 @@ def get_config(request: Request) -> ConfigResponse:
|
||||
return ConfigResponse(
|
||||
CONFIG_FILE_MOUNTED=cfg.CONFIG_FILE_MOUNTED,
|
||||
CONFIG_FILE_WRITABLE=cfg.CONFIG_FILE_WRITABLE,
|
||||
EXCLUDED_PLATFORMS=cfg.EXCLUDED_PLATFORMS,
|
||||
EXCLUDED_SINGLE_EXT=cfg.EXCLUDED_SINGLE_EXT,
|
||||
EXCLUDED_SINGLE_FILES=cfg.EXCLUDED_SINGLE_FILES,
|
||||
EXCLUDED_MULTI_FILES=cfg.EXCLUDED_MULTI_FILES,
|
||||
EXCLUDED_MULTI_PARTS_EXT=cfg.EXCLUDED_MULTI_PARTS_EXT,
|
||||
EXCLUDED_MULTI_PARTS_FILES=cfg.EXCLUDED_MULTI_PARTS_FILES,
|
||||
EXCLUDED_PLATFORMS=sorted(cfg.EXCLUDED_PLATFORMS),
|
||||
EXCLUDED_SINGLE_EXT=sorted(cfg.EXCLUDED_SINGLE_EXT),
|
||||
EXCLUDED_SINGLE_FILES=sorted(cfg.EXCLUDED_SINGLE_FILES),
|
||||
EXCLUDED_MULTI_FILES=sorted(cfg.EXCLUDED_MULTI_FILES),
|
||||
EXCLUDED_MULTI_PARTS_EXT=sorted(cfg.EXCLUDED_MULTI_PARTS_EXT),
|
||||
EXCLUDED_MULTI_PARTS_FILES=sorted(cfg.EXCLUDED_MULTI_PARTS_FILES),
|
||||
PLATFORMS_BINDING=cfg.PLATFORMS_BINDING,
|
||||
PLATFORMS_VERSIONS=cfg.PLATFORMS_VERSIONS,
|
||||
SKIP_HASH_CALCULATION=cfg.SKIP_HASH_CALCULATION,
|
||||
|
||||
@@ -413,7 +413,7 @@ async def _identify_rom(
|
||||
)
|
||||
|
||||
# Handle special media files from Screenscraper
|
||||
if _added_rom.ss_metadata:
|
||||
if _added_rom.ss_metadata and MetadataSource.SS in metadata_sources:
|
||||
preferred_media_types = get_preferred_media_types()
|
||||
for media_type in preferred_media_types:
|
||||
if _added_rom.ss_metadata.get(f"{media_type.value}_path"):
|
||||
@@ -423,7 +423,7 @@ async def _identify_rom(
|
||||
)
|
||||
|
||||
# Handle special media files from ES-DE gamelist.xml
|
||||
if _added_rom.gamelist_metadata:
|
||||
if _added_rom.gamelist_metadata and MetadataSource.GAMELIST in metadata_sources:
|
||||
preferred_media_types = get_preferred_media_types()
|
||||
for media_type in preferred_media_types:
|
||||
if _added_rom.gamelist_metadata.get(f"{media_type.value}_path"):
|
||||
@@ -433,7 +433,7 @@ async def _identify_rom(
|
||||
)
|
||||
|
||||
# Store normal and locked badges
|
||||
if _added_rom.ra_metadata:
|
||||
if _added_rom.ra_metadata and MetadataSource.RA in metadata_sources:
|
||||
for ach in _added_rom.ra_metadata.get("achievements", []):
|
||||
badge_url_lock = ach.get("badge_url_lock", None)
|
||||
badge_path_lock = ach.get("badge_path_lock", None)
|
||||
|
||||
@@ -75,11 +75,15 @@ class FSResourcesHandler(FSHandler):
|
||||
# Handle file:// URLs for gamelist.xml
|
||||
if url_cover.startswith("file://"):
|
||||
try:
|
||||
file_path = AnyioPath(url_cover[7:]) # Remove "file://" prefix
|
||||
if await file_path.exists():
|
||||
from handler.filesystem import fs_rom_handler
|
||||
|
||||
validated = fs_rom_handler.validate_path(
|
||||
url_cover[7:] # Remove "file://" prefix
|
||||
)
|
||||
if await AnyioPath(validated).exists():
|
||||
# Copy the file to the resources directory
|
||||
dest_path = f"{cover_file}/{size.value}.png"
|
||||
await self.copy_file(Path(str(file_path)), dest_path)
|
||||
await self.copy_file(validated, dest_path)
|
||||
|
||||
if ENABLE_SCHEDULED_CONVERT_IMAGES_TO_WEBP:
|
||||
self.image_converter.convert_to_webp(
|
||||
@@ -87,14 +91,13 @@ class FSResourcesHandler(FSHandler):
|
||||
force=True,
|
||||
)
|
||||
else:
|
||||
log.warning(f"Cover file not found: {file_path}")
|
||||
log.warning(f"Cover file not found: {str(validated)}")
|
||||
return None
|
||||
except Exception as exc:
|
||||
log.error(f"Unable to copy cover file {url_cover}: {str(exc)}")
|
||||
return None
|
||||
else:
|
||||
# Handle HTTP URLs
|
||||
# Validate URL to prevent SSRF attacks
|
||||
validate_url_for_http_request(url_cover, "url_cover")
|
||||
|
||||
httpx_client = ctx_httpx_client.get()
|
||||
@@ -103,6 +106,13 @@ class FSResourcesHandler(FSHandler):
|
||||
"GET", url_cover, timeout=120
|
||||
) as response:
|
||||
if response.status_code == status.HTTP_200_OK:
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
if not content_type.startswith("image/"):
|
||||
log.warning(
|
||||
f"Unexpected content type for cover: {content_type}"
|
||||
)
|
||||
return None
|
||||
|
||||
# Check if content is gzipped from response headers
|
||||
is_gzipped = (
|
||||
response.headers.get("content-encoding", "").lower()
|
||||
@@ -249,21 +259,23 @@ class FSResourcesHandler(FSHandler):
|
||||
# Handle file:// URLs for gamelist.xml
|
||||
if url_screenhot.startswith("file://"):
|
||||
try:
|
||||
file_path = AnyioPath(url_screenhot[7:]) # Remove "file://" prefix
|
||||
if await file_path.exists():
|
||||
from handler.filesystem import fs_rom_handler
|
||||
|
||||
validated = fs_rom_handler.validate_path(
|
||||
url_screenhot[7:] # Remove "file://" prefix
|
||||
)
|
||||
if await AnyioPath(validated).exists():
|
||||
# Copy the file to the resources directory
|
||||
await self.copy_file(
|
||||
Path(str(file_path)), f"{screenshot_path}/{idx}.jpg"
|
||||
)
|
||||
await self.copy_file(validated, f"{screenshot_path}/{idx}.jpg")
|
||||
else:
|
||||
log.warning(f"Screenshot file not found: {file_path}")
|
||||
log.warning(f"Screenshot file not found: {str(validated)}")
|
||||
return None
|
||||
except Exception as exc:
|
||||
log.error(f"Unable to copy screenshot file {url_screenhot}: {str(exc)}")
|
||||
return None
|
||||
else:
|
||||
# Handle HTTP URLs
|
||||
# Validate URL to prevent SSRF attacks
|
||||
# Validate to prevent SSRF attacks
|
||||
validate_url_for_http_request(url_screenhot, "url_screenshot")
|
||||
|
||||
httpx_client = ctx_httpx_client.get()
|
||||
@@ -272,6 +284,13 @@ class FSResourcesHandler(FSHandler):
|
||||
"GET", url_screenhot, timeout=120
|
||||
) as response:
|
||||
if response.status_code == status.HTTP_200_OK:
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
if not content_type.startswith("image/"):
|
||||
log.warning(
|
||||
f"Unexpected content type for screenshot: {content_type}"
|
||||
)
|
||||
return None
|
||||
|
||||
# Check if content is gzipped from response headers
|
||||
is_gzipped = (
|
||||
response.headers.get("content-encoding", "").lower()
|
||||
@@ -365,21 +384,22 @@ class FSResourcesHandler(FSHandler):
|
||||
# Handle file:// URLs for gamelist.xml
|
||||
if url_manual.startswith("file://"):
|
||||
try:
|
||||
file_path = AnyioPath(url_manual[7:]) # Remove "file://" prefix
|
||||
if await file_path.exists():
|
||||
from handler.filesystem import fs_rom_handler
|
||||
|
||||
validated = fs_rom_handler.validate_path(
|
||||
url_manual[7:] # Remove "file://" prefix
|
||||
)
|
||||
if await AnyioPath(validated).exists():
|
||||
# Copy the file to the resources directory
|
||||
await self.copy_file(
|
||||
Path(str(file_path)), f"{manual_path}/{rom.id}.pdf"
|
||||
)
|
||||
await self.copy_file(validated, f"{manual_path}/{rom.id}.pdf")
|
||||
else:
|
||||
log.warning(f"Manual file not found: {file_path}")
|
||||
log.warning(f"Manual file not found: {str(validated)}")
|
||||
return None
|
||||
except Exception as exc:
|
||||
log.error(f"Unable to copy manual file {url_manual}: {str(exc)}")
|
||||
return None
|
||||
else:
|
||||
# Handle HTTP URL
|
||||
# Validate URL to prevent SSRF attacks
|
||||
validate_url_for_http_request(url_manual, "url_manual")
|
||||
|
||||
httpx_client = ctx_httpx_client.get()
|
||||
@@ -388,6 +408,13 @@ class FSResourcesHandler(FSHandler):
|
||||
"GET", url_manual, timeout=120
|
||||
) as response:
|
||||
if response.status_code == status.HTTP_200_OK:
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
if not content_type.startswith("application/pdf"):
|
||||
log.warning(
|
||||
f"Unexpected content type for manual: {content_type}"
|
||||
)
|
||||
return None
|
||||
|
||||
# Check if content is gzipped from response headers
|
||||
is_gzipped = (
|
||||
response.headers.get("content-encoding", "").lower()
|
||||
@@ -440,7 +467,6 @@ class FSResourcesHandler(FSHandler):
|
||||
|
||||
# Retroachievements
|
||||
async def store_ra_badge(self, url: str, path: str) -> None:
|
||||
# Validate URL to prevent SSRF attacks
|
||||
validate_url_for_http_request(url, "url_badge")
|
||||
|
||||
httpx_client = ctx_httpx_client.get()
|
||||
@@ -456,6 +482,13 @@ class FSResourcesHandler(FSHandler):
|
||||
try:
|
||||
async with httpx_client.stream("GET", url, timeout=120) as response:
|
||||
if response.status_code == status.HTTP_200_OK:
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
if not content_type.startswith("image/"):
|
||||
log.warning(
|
||||
f"Unexpected content type for badge: {content_type}"
|
||||
)
|
||||
return
|
||||
|
||||
async with await self.write_file_streamed(
|
||||
path=directory, filename=filename
|
||||
) as f:
|
||||
@@ -498,7 +531,12 @@ class FSResourcesHandler(FSHandler):
|
||||
# Handle file:// URLs for gamelist.xml
|
||||
if url_media.startswith("file://"):
|
||||
try:
|
||||
file_path = AnyioPath(url_media[7:]) # Remove "file://" prefix
|
||||
from handler.filesystem import fs_rom_handler
|
||||
|
||||
validated = fs_rom_handler.validate_path(
|
||||
url_media[7:] # Remove "file://" prefix
|
||||
)
|
||||
file_path = AnyioPath(validated)
|
||||
if await file_path.exists():
|
||||
await self.copy_file(Path(str(file_path)), dest_path)
|
||||
except Exception as exc:
|
||||
@@ -506,7 +544,6 @@ class FSResourcesHandler(FSHandler):
|
||||
return None
|
||||
else:
|
||||
# Handle HTTP URLs
|
||||
# Validate URL to prevent SSRF attacks
|
||||
validate_url_for_http_request(url_media, "url_media")
|
||||
|
||||
httpx_client = ctx_httpx_client.get()
|
||||
@@ -515,6 +552,17 @@ class FSResourcesHandler(FSHandler):
|
||||
"GET", url_media, timeout=120
|
||||
) as response:
|
||||
if response.status_code == status.HTTP_200_OK:
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
if not (
|
||||
content_type.startswith("image/")
|
||||
or content_type.startswith("video/")
|
||||
or content_type.startswith("application/pdf")
|
||||
):
|
||||
log.warning(
|
||||
f"Unexpected content type for media: {content_type}"
|
||||
)
|
||||
return None
|
||||
|
||||
async with await self.write_file_streamed(
|
||||
path=directory, filename=filename
|
||||
) as f:
|
||||
|
||||
@@ -745,6 +745,6 @@ class FSRomsHandler(FSHandler):
|
||||
if platform_slug == UPS.PICO and fs_name.lower().endswith(
|
||||
PICO8_CARTRIDGE_EXTENSION
|
||||
):
|
||||
rom_path = self.validate_path(f"{fs_path}/{fs_name}")
|
||||
return f"file://{rom_path}"
|
||||
self.validate_path(f"{fs_path}/{fs_name}")
|
||||
return f"file://{fs_path}/{fs_name}"
|
||||
return None
|
||||
|
||||
@@ -1 +1 @@
|
||||
https://howlongtobeat.com/api/finder
|
||||
https://howlongtobeat.com/api/find
|
||||
|
||||
@@ -99,10 +99,9 @@ XML_TAG_MAP: Final = {
|
||||
|
||||
def _make_file_uri(platform_dir: str, raw_text: str) -> str:
|
||||
cleaned_text = raw_text.replace("./", "")
|
||||
validated_path = fs_platform_handler.validate_path(
|
||||
os.path.join(platform_dir, cleaned_text)
|
||||
)
|
||||
return f"file://{str(validated_path)}"
|
||||
joined_path = os.path.join(platform_dir, cleaned_text)
|
||||
fs_platform_handler.validate_path(joined_path)
|
||||
return f"file://{joined_path}"
|
||||
|
||||
|
||||
def extract_media_from_gamelist_rom(
|
||||
@@ -148,7 +147,9 @@ def extract_media_from_gamelist_rom(
|
||||
found_files = glob.glob(str(search_path))
|
||||
if found_files:
|
||||
# trunk-ignore(mypy/literal-required)
|
||||
gamelist_media[media_key] = f"file://{str(found_files[0])}"
|
||||
gamelist_media[media_key] = (
|
||||
f"file://{str(Path(found_files[0]).relative_to(fs_platform_handler.base_path))}"
|
||||
)
|
||||
|
||||
return gamelist_media
|
||||
|
||||
|
||||
@@ -178,9 +178,11 @@ class HLTBHandler(MetadataHandler):
|
||||
self.base_url = "https://howlongtobeat.com"
|
||||
self.user_endpoint = f"{self.base_url}/api/user"
|
||||
self.stats_endpoint = f"{self.base_url}/api/stats/games?platform=1&year=2000"
|
||||
self.search_url = f"{self.base_url}/api/search"
|
||||
self.search_url = f"{self.base_url}/api/find"
|
||||
self.search_init_url = f"{self.search_url}/init"
|
||||
self.security_token = None
|
||||
self.hp_key = None
|
||||
self.hp_val = None
|
||||
self.min_similarity_score: Final = 0.85
|
||||
|
||||
# HLTB rotates their search endpoint regularly
|
||||
@@ -226,7 +228,10 @@ class HLTBHandler(MetadataHandler):
|
||||
timeout=10,
|
||||
)
|
||||
response.raise_for_status()
|
||||
self.security_token = response.json().get("token", None)
|
||||
data = response.json()
|
||||
self.security_token = data.get("token", None)
|
||||
self.hp_key = data.get("hpKey", None)
|
||||
self.hp_val = data.get("hpVal", None)
|
||||
except Exception as e:
|
||||
log.warning("Unexpected error fetching HLTB security token: %s", e)
|
||||
|
||||
@@ -262,9 +267,12 @@ class HLTBHandler(MetadataHandler):
|
||||
"Content-Type": "application/json",
|
||||
"Referer": "https://howlongtobeat.com",
|
||||
"User-Agent": f"RomM/{get_version()}",
|
||||
"X-Auth-Token": self.security_token,
|
||||
"x-auth-token": self.security_token,
|
||||
"x-hp-key": self.hp_key,
|
||||
"x-hp-val": self.hp_val,
|
||||
}
|
||||
|
||||
payload[self.hp_key] = self.hp_val
|
||||
log.debug(
|
||||
"HowLongToBeat API request: URL=%s, Headers=%s, Payload=%s, Timeout=%s",
|
||||
url,
|
||||
|
||||
@@ -14,12 +14,25 @@ def test_config_loader():
|
||||
os.path.join(Path(__file__).resolve().parent, "fixtures", "config/config.yml")
|
||||
)
|
||||
|
||||
assert loader.config.EXCLUDED_PLATFORMS == ["romm"]
|
||||
assert loader.config.EXCLUDED_SINGLE_EXT == ["xml"]
|
||||
assert loader.config.EXCLUDED_SINGLE_FILES == ["info.txt"]
|
||||
assert loader.config.EXCLUDED_MULTI_FILES == ["my_multi_file_game", "DLC"]
|
||||
assert loader.config.EXCLUDED_MULTI_PARTS_EXT == ["txt"]
|
||||
assert loader.config.EXCLUDED_MULTI_PARTS_FILES == ["data.xml"]
|
||||
assert loader.config.EXCLUDED_PLATFORMS == {*DEFAULT_EXCLUDED_DIRS, "romm"}
|
||||
assert loader.config.EXCLUDED_SINGLE_EXT == {
|
||||
*(e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS),
|
||||
"xml",
|
||||
}
|
||||
assert loader.config.EXCLUDED_SINGLE_FILES == {*DEFAULT_EXCLUDED_FILES, "info.txt"}
|
||||
assert loader.config.EXCLUDED_MULTI_FILES == {
|
||||
*DEFAULT_EXCLUDED_DIRS,
|
||||
"my_multi_file_game",
|
||||
"DLC",
|
||||
}
|
||||
assert loader.config.EXCLUDED_MULTI_PARTS_EXT == {
|
||||
*(e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS),
|
||||
"txt",
|
||||
}
|
||||
assert loader.config.EXCLUDED_MULTI_PARTS_FILES == {
|
||||
*DEFAULT_EXCLUDED_FILES,
|
||||
"data.xml",
|
||||
}
|
||||
assert loader.config.PLATFORMS_BINDING == {"gc": "ngc"}
|
||||
assert loader.config.PLATFORMS_VERSIONS == {"naomi": "arcade"}
|
||||
assert loader.config.ROMS_FOLDER_NAME == "ROMS"
|
||||
@@ -63,16 +76,16 @@ def test_empty_config_loader():
|
||||
)
|
||||
)
|
||||
|
||||
assert loader.config.EXCLUDED_PLATFORMS == DEFAULT_EXCLUDED_DIRS
|
||||
assert loader.config.EXCLUDED_SINGLE_EXT == [
|
||||
assert loader.config.EXCLUDED_PLATFORMS == set(DEFAULT_EXCLUDED_DIRS)
|
||||
assert loader.config.EXCLUDED_SINGLE_EXT == {
|
||||
e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS
|
||||
]
|
||||
assert loader.config.EXCLUDED_SINGLE_FILES == DEFAULT_EXCLUDED_FILES
|
||||
assert loader.config.EXCLUDED_MULTI_FILES == DEFAULT_EXCLUDED_DIRS
|
||||
assert loader.config.EXCLUDED_MULTI_PARTS_EXT == [
|
||||
}
|
||||
assert loader.config.EXCLUDED_SINGLE_FILES == set(DEFAULT_EXCLUDED_FILES)
|
||||
assert loader.config.EXCLUDED_MULTI_FILES == set(DEFAULT_EXCLUDED_DIRS)
|
||||
assert loader.config.EXCLUDED_MULTI_PARTS_EXT == {
|
||||
e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS
|
||||
]
|
||||
assert loader.config.EXCLUDED_MULTI_PARTS_FILES == DEFAULT_EXCLUDED_FILES
|
||||
}
|
||||
assert loader.config.EXCLUDED_MULTI_PARTS_FILES == set(DEFAULT_EXCLUDED_FILES)
|
||||
assert loader.config.PLATFORMS_BINDING == {}
|
||||
assert loader.config.PLATFORMS_VERSIONS == {}
|
||||
assert loader.config.ROMS_FOLDER_NAME == "roms"
|
||||
|
||||
@@ -263,7 +263,7 @@ class TestGetPico8CoverUrl:
|
||||
fs_name="mygame.p8.png",
|
||||
fs_path="pico/roms",
|
||||
)
|
||||
expected = f"file://{Path(LIBRARY_BASE_PATH).resolve() / 'pico/roms' / 'mygame.p8.png'}"
|
||||
expected = f"file://pico/roms/mygame.p8.png"
|
||||
assert url == expected
|
||||
|
||||
def test_returns_none_for_non_pico8_platform(self, handler: FSRomsHandler):
|
||||
|
||||
@@ -24,16 +24,16 @@ def test_config(client):
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
|
||||
config = response.json()
|
||||
assert config.get("EXCLUDED_PLATFORMS") == DEFAULT_EXCLUDED_DIRS
|
||||
assert config.get("EXCLUDED_SINGLE_EXT") == [
|
||||
assert config.get("EXCLUDED_PLATFORMS") == sorted(DEFAULT_EXCLUDED_DIRS)
|
||||
assert config.get("EXCLUDED_SINGLE_EXT") == sorted(
|
||||
e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS
|
||||
]
|
||||
assert config.get("EXCLUDED_SINGLE_FILES") == DEFAULT_EXCLUDED_FILES
|
||||
assert config.get("EXCLUDED_MULTI_FILES") == DEFAULT_EXCLUDED_DIRS
|
||||
assert config.get("EXCLUDED_MULTI_PARTS_EXT") == [
|
||||
)
|
||||
assert config.get("EXCLUDED_SINGLE_FILES") == sorted(DEFAULT_EXCLUDED_FILES)
|
||||
assert config.get("EXCLUDED_MULTI_FILES") == sorted(DEFAULT_EXCLUDED_DIRS)
|
||||
assert config.get("EXCLUDED_MULTI_PARTS_EXT") == sorted(
|
||||
e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS
|
||||
]
|
||||
assert config.get("EXCLUDED_MULTI_PARTS_FILES") == DEFAULT_EXCLUDED_FILES
|
||||
)
|
||||
assert config.get("EXCLUDED_MULTI_PARTS_FILES") == sorted(DEFAULT_EXCLUDED_FILES)
|
||||
assert config.get("PLATFORMS_BINDING") == {}
|
||||
assert not config.get("SKIP_HASH_CALCULATION")
|
||||
|
||||
|
||||
@@ -151,8 +151,8 @@ class TestFSHandler:
|
||||
|
||||
# Mock configuration
|
||||
with patch("handler.filesystem.base_handler.cm.get_config") as mock_config:
|
||||
mock_config.return_value.EXCLUDED_SINGLE_EXT = ["tmp"]
|
||||
mock_config.return_value.EXCLUDED_SINGLE_FILES = ["test.txt"]
|
||||
mock_config.return_value.EXCLUDED_SINGLE_EXT = {"tmp"}
|
||||
mock_config.return_value.EXCLUDED_SINGLE_FILES = {"test.txt"}
|
||||
|
||||
result = handler.exclude_single_files(files)
|
||||
|
||||
|
||||
@@ -22,12 +22,12 @@ class TestFSFirmwareHandler:
|
||||
@pytest.fixture
|
||||
def config(self):
|
||||
return Config(
|
||||
EXCLUDED_PLATFORMS=[],
|
||||
EXCLUDED_SINGLE_EXT=["tmp"],
|
||||
EXCLUDED_SINGLE_FILES=[],
|
||||
EXCLUDED_MULTI_FILES=[],
|
||||
EXCLUDED_MULTI_PARTS_EXT=[],
|
||||
EXCLUDED_MULTI_PARTS_FILES=[],
|
||||
EXCLUDED_PLATFORMS=set(),
|
||||
EXCLUDED_SINGLE_EXT={"tmp"},
|
||||
EXCLUDED_SINGLE_FILES=set(),
|
||||
EXCLUDED_MULTI_FILES=set(),
|
||||
EXCLUDED_MULTI_PARTS_EXT=set(),
|
||||
EXCLUDED_MULTI_PARTS_FILES=set(),
|
||||
PLATFORMS_BINDING={},
|
||||
PLATFORMS_VERSIONS={},
|
||||
ROMS_FOLDER_NAME="roms",
|
||||
|
||||
@@ -17,12 +17,12 @@ class TestFSPlatformsHandler:
|
||||
@pytest.fixture
|
||||
def config(self):
|
||||
return Config(
|
||||
EXCLUDED_PLATFORMS=["romm", "excluded_platform"],
|
||||
EXCLUDED_SINGLE_EXT=["tmp"],
|
||||
EXCLUDED_SINGLE_FILES=[],
|
||||
EXCLUDED_MULTI_FILES=[],
|
||||
EXCLUDED_MULTI_PARTS_EXT=[],
|
||||
EXCLUDED_MULTI_PARTS_FILES=[],
|
||||
EXCLUDED_PLATFORMS={"romm", "excluded_platform"},
|
||||
EXCLUDED_SINGLE_EXT={"tmp"},
|
||||
EXCLUDED_SINGLE_FILES=set(),
|
||||
EXCLUDED_MULTI_FILES=set(),
|
||||
EXCLUDED_MULTI_PARTS_EXT=set(),
|
||||
EXCLUDED_MULTI_PARTS_FILES=set(),
|
||||
PLATFORMS_BINDING={},
|
||||
PLATFORMS_VERSIONS={},
|
||||
ROMS_FOLDER_NAME="roms",
|
||||
@@ -32,12 +32,12 @@ class TestFSPlatformsHandler:
|
||||
@pytest.fixture
|
||||
def config_custom_folder(self):
|
||||
return Config(
|
||||
EXCLUDED_PLATFORMS=[],
|
||||
EXCLUDED_SINGLE_EXT=[],
|
||||
EXCLUDED_SINGLE_FILES=[],
|
||||
EXCLUDED_MULTI_FILES=[],
|
||||
EXCLUDED_MULTI_PARTS_EXT=[],
|
||||
EXCLUDED_MULTI_PARTS_FILES=[],
|
||||
EXCLUDED_PLATFORMS=set(),
|
||||
EXCLUDED_SINGLE_EXT=set(),
|
||||
EXCLUDED_SINGLE_FILES=set(),
|
||||
EXCLUDED_MULTI_FILES=set(),
|
||||
EXCLUDED_MULTI_PARTS_EXT=set(),
|
||||
EXCLUDED_MULTI_PARTS_FILES=set(),
|
||||
PLATFORMS_BINDING={},
|
||||
PLATFORMS_VERSIONS={},
|
||||
ROMS_FOLDER_NAME="ROMS",
|
||||
@@ -194,7 +194,7 @@ class TestFSPlatformsHandler:
|
||||
with patch(
|
||||
"handler.filesystem.platforms_handler.cm.get_config", return_value=config
|
||||
):
|
||||
config.EXCLUDED_PLATFORMS = ["psx"]
|
||||
config.EXCLUDED_PLATFORMS = {"psx"}
|
||||
result = await handler.get_platforms()
|
||||
|
||||
assert "n64" in result
|
||||
|
||||
@@ -26,12 +26,12 @@ class TestFSRomsHandler:
|
||||
@pytest.fixture
|
||||
def config(self):
|
||||
return Config(
|
||||
EXCLUDED_PLATFORMS=[],
|
||||
EXCLUDED_SINGLE_EXT=["tmp"],
|
||||
EXCLUDED_SINGLE_FILES=["excluded_test.tmp"],
|
||||
EXCLUDED_MULTI_FILES=["excluded_multi"],
|
||||
EXCLUDED_MULTI_PARTS_EXT=["tmp"],
|
||||
EXCLUDED_MULTI_PARTS_FILES=["excluded_part.bin"],
|
||||
EXCLUDED_PLATFORMS=set(),
|
||||
EXCLUDED_SINGLE_EXT={"tmp"},
|
||||
EXCLUDED_SINGLE_FILES={"excluded_test.tmp"},
|
||||
EXCLUDED_MULTI_FILES={"excluded_multi"},
|
||||
EXCLUDED_MULTI_PARTS_EXT={"tmp"},
|
||||
EXCLUDED_MULTI_PARTS_FILES={"excluded_part.bin"},
|
||||
PLATFORMS_BINDING={},
|
||||
PLATFORMS_VERSIONS={},
|
||||
ROMS_FOLDER_NAME="roms",
|
||||
@@ -107,12 +107,12 @@ class TestFSRomsHandler:
|
||||
m.setattr(
|
||||
"handler.filesystem.roms_handler.cm.get_config",
|
||||
lambda: Config(
|
||||
EXCLUDED_PLATFORMS=[],
|
||||
EXCLUDED_SINGLE_EXT=[],
|
||||
EXCLUDED_SINGLE_FILES=[],
|
||||
EXCLUDED_MULTI_FILES=[],
|
||||
EXCLUDED_MULTI_PARTS_EXT=[],
|
||||
EXCLUDED_MULTI_PARTS_FILES=[],
|
||||
EXCLUDED_PLATFORMS=set(),
|
||||
EXCLUDED_SINGLE_EXT=set(),
|
||||
EXCLUDED_SINGLE_FILES=set(),
|
||||
EXCLUDED_MULTI_FILES=set(),
|
||||
EXCLUDED_MULTI_PARTS_EXT=set(),
|
||||
EXCLUDED_MULTI_PARTS_FILES=set(),
|
||||
PLATFORMS_BINDING={},
|
||||
PLATFORMS_VERSIONS={},
|
||||
ROMS_FOLDER_NAME="roms",
|
||||
@@ -134,12 +134,12 @@ class TestFSRomsHandler:
|
||||
m.setattr(
|
||||
"handler.filesystem.roms_handler.cm.get_config",
|
||||
lambda: Config(
|
||||
EXCLUDED_PLATFORMS=[],
|
||||
EXCLUDED_SINGLE_EXT=[],
|
||||
EXCLUDED_SINGLE_FILES=[],
|
||||
EXCLUDED_MULTI_FILES=[],
|
||||
EXCLUDED_MULTI_PARTS_EXT=[],
|
||||
EXCLUDED_MULTI_PARTS_FILES=[],
|
||||
EXCLUDED_PLATFORMS=set(),
|
||||
EXCLUDED_SINGLE_EXT=set(),
|
||||
EXCLUDED_SINGLE_FILES=set(),
|
||||
EXCLUDED_MULTI_FILES=set(),
|
||||
EXCLUDED_MULTI_PARTS_EXT=set(),
|
||||
EXCLUDED_MULTI_PARTS_FILES=set(),
|
||||
PLATFORMS_BINDING={},
|
||||
PLATFORMS_VERSIONS={},
|
||||
ROMS_FOLDER_NAME="roms",
|
||||
@@ -237,12 +237,12 @@ class TestFSRomsHandler:
|
||||
"""Test exclude_multi_roms with no exclusions"""
|
||||
roms = ["Game1", "Game2", "Game3"]
|
||||
config = Config(
|
||||
EXCLUDED_PLATFORMS=[],
|
||||
EXCLUDED_SINGLE_EXT=[],
|
||||
EXCLUDED_SINGLE_FILES=[],
|
||||
EXCLUDED_MULTI_FILES=[],
|
||||
EXCLUDED_MULTI_PARTS_EXT=[],
|
||||
EXCLUDED_MULTI_PARTS_FILES=[],
|
||||
EXCLUDED_PLATFORMS=set(),
|
||||
EXCLUDED_SINGLE_EXT=set(),
|
||||
EXCLUDED_SINGLE_FILES=set(),
|
||||
EXCLUDED_MULTI_FILES=set(),
|
||||
EXCLUDED_MULTI_PARTS_EXT=set(),
|
||||
EXCLUDED_MULTI_PARTS_FILES=set(),
|
||||
PLATFORMS_BINDING={},
|
||||
PLATFORMS_VERSIONS={},
|
||||
ROMS_FOLDER_NAME="roms",
|
||||
|
||||
Reference in New Issue
Block a user