From ec8583016b90abf2b43031aeda868628b61b9fa4 Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Fri, 3 Apr 2026 10:50:38 -0400 Subject: [PATCH] mega ton of fixes for 4.8 --- Dockerfile | 2 +- backend/config/config_manager.py | 117 ++++++++++-------- backend/endpoints/configs.py | 12 +- backend/endpoints/sockets/scan.py | 6 +- .../handler/filesystem/resources_handler.py | 92 ++++++++++---- backend/handler/filesystem/roms_handler.py | 4 +- .../handler/metadata/fixtures/hltb_api_url | 2 +- backend/handler/metadata/gamelist_handler.py | 11 +- backend/handler/metadata/hltb_handler.py | 14 ++- backend/tests/config/test_config_loader.py | 41 +++--- backend/tests/endpoints/sockets/test_scan.py | 2 +- backend/tests/endpoints/test_config.py | 16 +-- .../handler/filesystem/test_base_handler.py | 4 +- .../filesystem/test_firmware_handler.py | 12 +- .../filesystem/test_platforms_handler.py | 26 ++-- .../handler/filesystem/test_roms_handler.py | 48 +++---- 16 files changed, 248 insertions(+), 161 deletions(-) diff --git a/Dockerfile b/Dockerfile index 62fdc0ba9..61ceb260e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,7 +58,7 @@ RUN npm install WORKDIR /app # Install uv for the non-root user -COPY --from=ghcr.io/astral-sh/uv:0.7.19 /uv /uvx /usr/local/bin/ +COPY --from=ghcr.io/astral-sh/uv:0.11.2 /uv /uvx /usr/local/bin/ # Install Python RUN uv python install 3.13 diff --git a/backend/config/config_manager.py b/backend/config/config_manager.py index 2755c357b..9b044308c 100644 --- a/backend/config/config_manager.py +++ b/backend/config/config_manager.py @@ -45,6 +45,7 @@ DEFAULT_EXCLUDED_FILES: Final = [ ".stfolder", "@SynoResource", "gamelist.xml", + "metadata.pegasus.xml", ] DEFAULT_EXCLUDED_DIRS: Final = [ "@eaDir", @@ -100,12 +101,12 @@ class NetplayICEServer(TypedDict): class Config: CONFIG_FILE_MOUNTED: bool CONFIG_FILE_WRITABLE: bool - EXCLUDED_PLATFORMS: list[str] - EXCLUDED_SINGLE_EXT: list[str] - EXCLUDED_SINGLE_FILES: list[str] - EXCLUDED_MULTI_FILES: list[str] - EXCLUDED_MULTI_PARTS_EXT: list[str] - EXCLUDED_MULTI_PARTS_FILES: list[str] + EXCLUDED_PLATFORMS: set[str] + EXCLUDED_SINGLE_EXT: set[str] + EXCLUDED_SINGLE_FILES: set[str] + EXCLUDED_MULTI_FILES: set[str] + EXCLUDED_MULTI_PARTS_EXT: set[str] + EXCLUDED_MULTI_PARTS_FILES: set[str] GAMELIST_AUTO_EXPORT_ON_SCAN: bool PLATFORMS_BINDING: dict[str, str] PLATFORMS_VERSIONS: dict[str, str] @@ -224,40 +225,56 @@ class ConfigManager: self.config = Config( CONFIG_FILE_MOUNTED=self._config_file_mounted, CONFIG_FILE_WRITABLE=self._config_file_writable, - EXCLUDED_PLATFORMS=pydash.get( - self._raw_config, "exclude.platforms", DEFAULT_EXCLUDED_DIRS - ), - EXCLUDED_SINGLE_EXT=[ - e.lower() - for e in pydash.get( + EXCLUDED_PLATFORMS={ + *DEFAULT_EXCLUDED_DIRS, + *pydash.get(self._raw_config, "exclude.platforms", []), + }, + EXCLUDED_SINGLE_EXT={ + *(e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS), + *( + e.lower() + for e in pydash.get( + self._raw_config, + "exclude.roms.single_file.extensions", + [], + ) + ), + }, + EXCLUDED_SINGLE_FILES={ + *DEFAULT_EXCLUDED_FILES, + *pydash.get( self._raw_config, - "exclude.roms.single_file.extensions", - DEFAULT_EXCLUDED_EXTENSIONS, - ) - ], - EXCLUDED_SINGLE_FILES=pydash.get( - self._raw_config, - "exclude.roms.single_file.names", - DEFAULT_EXCLUDED_FILES, - ), - EXCLUDED_MULTI_FILES=pydash.get( - self._raw_config, - "exclude.roms.multi_file.names", - DEFAULT_EXCLUDED_DIRS, - ), - EXCLUDED_MULTI_PARTS_EXT=[ - e.lower() - for e in pydash.get( + "exclude.roms.single_file.names", + [], + ), + }, + EXCLUDED_MULTI_FILES={ + *DEFAULT_EXCLUDED_DIRS, + *pydash.get( self._raw_config, - "exclude.roms.multi_file.parts.extensions", - DEFAULT_EXCLUDED_EXTENSIONS, - ) - ], - EXCLUDED_MULTI_PARTS_FILES=pydash.get( - self._raw_config, - "exclude.roms.multi_file.parts.names", - DEFAULT_EXCLUDED_FILES, - ), + "exclude.roms.multi_file.names", + [], + ), + }, + EXCLUDED_MULTI_PARTS_EXT={ + *(e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS), + *( + e.lower() + for e in pydash.get( + self._raw_config, + "exclude.roms.multi_file.parts.extensions", + [], + ) + ), + }, + EXCLUDED_MULTI_PARTS_FILES={ + *DEFAULT_EXCLUDED_FILES, + *pydash.get( + self._raw_config, + "exclude.roms.multi_file.parts.names", + [], + ), + }, PLATFORMS_BINDING=pydash.get(self._raw_config, "system.platforms", {}), PLATFORMS_VERSIONS=pydash.get(self._raw_config, "system.versions", {}), ROMS_FOLDER_NAME=pydash.get( @@ -377,35 +394,35 @@ class ConfigManager: def _validate_config(self): """Validates the config.yml file""" - if not isinstance(self.config.EXCLUDED_PLATFORMS, list): + if not isinstance(self.config.EXCLUDED_PLATFORMS, set): log.critical("Invalid config.yml: exclude.platforms must be a list") sys.exit(3) - if not isinstance(self.config.EXCLUDED_SINGLE_EXT, list): + if not isinstance(self.config.EXCLUDED_SINGLE_EXT, set): log.critical( "Invalid config.yml: exclude.roms.single_file.extensions must be a list" ) sys.exit(3) - if not isinstance(self.config.EXCLUDED_SINGLE_FILES, list): + if not isinstance(self.config.EXCLUDED_SINGLE_FILES, set): log.critical( "Invalid config.yml: exclude.roms.single_file.names must be a list" ) sys.exit(3) - if not isinstance(self.config.EXCLUDED_MULTI_FILES, list): + if not isinstance(self.config.EXCLUDED_MULTI_FILES, set): log.critical( "Invalid config.yml: exclude.roms.multi_file.names must be a list" ) sys.exit(3) - if not isinstance(self.config.EXCLUDED_MULTI_PARTS_EXT, list): + if not isinstance(self.config.EXCLUDED_MULTI_PARTS_EXT, set): log.critical( "Invalid config.yml: exclude.roms.multi_file.parts.extensions must be a list" ) sys.exit(3) - if not isinstance(self.config.EXCLUDED_MULTI_PARTS_FILES, list): + if not isinstance(self.config.EXCLUDED_MULTI_PARTS_FILES, set): log.critical( "Invalid config.yml: exclude.roms.multi_file.parts.names must be a list" ) @@ -576,17 +593,17 @@ class ConfigManager: self._raw_config = { "exclude": { - "platforms": self.config.EXCLUDED_PLATFORMS, + "platforms": sorted(self.config.EXCLUDED_PLATFORMS), "roms": { "single_file": { - "extensions": self.config.EXCLUDED_SINGLE_EXT, - "names": self.config.EXCLUDED_SINGLE_FILES, + "extensions": sorted(self.config.EXCLUDED_SINGLE_EXT), + "names": sorted(self.config.EXCLUDED_SINGLE_FILES), }, "multi_file": { - "names": self.config.EXCLUDED_MULTI_FILES, + "names": sorted(self.config.EXCLUDED_MULTI_FILES), "parts": { - "extensions": self.config.EXCLUDED_MULTI_PARTS_EXT, - "names": self.config.EXCLUDED_MULTI_PARTS_FILES, + "extensions": sorted(self.config.EXCLUDED_MULTI_PARTS_EXT), + "names": sorted(self.config.EXCLUDED_MULTI_PARTS_FILES), }, }, }, diff --git a/backend/endpoints/configs.py b/backend/endpoints/configs.py index 65482f5f8..40d14fae4 100644 --- a/backend/endpoints/configs.py +++ b/backend/endpoints/configs.py @@ -37,12 +37,12 @@ def get_config(request: Request) -> ConfigResponse: return ConfigResponse( CONFIG_FILE_MOUNTED=cfg.CONFIG_FILE_MOUNTED, CONFIG_FILE_WRITABLE=cfg.CONFIG_FILE_WRITABLE, - EXCLUDED_PLATFORMS=cfg.EXCLUDED_PLATFORMS, - EXCLUDED_SINGLE_EXT=cfg.EXCLUDED_SINGLE_EXT, - EXCLUDED_SINGLE_FILES=cfg.EXCLUDED_SINGLE_FILES, - EXCLUDED_MULTI_FILES=cfg.EXCLUDED_MULTI_FILES, - EXCLUDED_MULTI_PARTS_EXT=cfg.EXCLUDED_MULTI_PARTS_EXT, - EXCLUDED_MULTI_PARTS_FILES=cfg.EXCLUDED_MULTI_PARTS_FILES, + EXCLUDED_PLATFORMS=sorted(cfg.EXCLUDED_PLATFORMS), + EXCLUDED_SINGLE_EXT=sorted(cfg.EXCLUDED_SINGLE_EXT), + EXCLUDED_SINGLE_FILES=sorted(cfg.EXCLUDED_SINGLE_FILES), + EXCLUDED_MULTI_FILES=sorted(cfg.EXCLUDED_MULTI_FILES), + EXCLUDED_MULTI_PARTS_EXT=sorted(cfg.EXCLUDED_MULTI_PARTS_EXT), + EXCLUDED_MULTI_PARTS_FILES=sorted(cfg.EXCLUDED_MULTI_PARTS_FILES), PLATFORMS_BINDING=cfg.PLATFORMS_BINDING, PLATFORMS_VERSIONS=cfg.PLATFORMS_VERSIONS, SKIP_HASH_CALCULATION=cfg.SKIP_HASH_CALCULATION, diff --git a/backend/endpoints/sockets/scan.py b/backend/endpoints/sockets/scan.py index 46107f1dc..ed21d5cce 100644 --- a/backend/endpoints/sockets/scan.py +++ b/backend/endpoints/sockets/scan.py @@ -413,7 +413,7 @@ async def _identify_rom( ) # Handle special media files from Screenscraper - if _added_rom.ss_metadata: + if _added_rom.ss_metadata and MetadataSource.SS in metadata_sources: preferred_media_types = get_preferred_media_types() for media_type in preferred_media_types: if _added_rom.ss_metadata.get(f"{media_type.value}_path"): @@ -423,7 +423,7 @@ async def _identify_rom( ) # Handle special media files from ES-DE gamelist.xml - if _added_rom.gamelist_metadata: + if _added_rom.gamelist_metadata and MetadataSource.GAMELIST in metadata_sources: preferred_media_types = get_preferred_media_types() for media_type in preferred_media_types: if _added_rom.gamelist_metadata.get(f"{media_type.value}_path"): @@ -433,7 +433,7 @@ async def _identify_rom( ) # Store normal and locked badges - if _added_rom.ra_metadata: + if _added_rom.ra_metadata and MetadataSource.RA in metadata_sources: for ach in _added_rom.ra_metadata.get("achievements", []): badge_url_lock = ach.get("badge_url_lock", None) badge_path_lock = ach.get("badge_path_lock", None) diff --git a/backend/handler/filesystem/resources_handler.py b/backend/handler/filesystem/resources_handler.py index 2b2b334f4..20e2ab9ea 100644 --- a/backend/handler/filesystem/resources_handler.py +++ b/backend/handler/filesystem/resources_handler.py @@ -75,11 +75,15 @@ class FSResourcesHandler(FSHandler): # Handle file:// URLs for gamelist.xml if url_cover.startswith("file://"): try: - file_path = AnyioPath(url_cover[7:]) # Remove "file://" prefix - if await file_path.exists(): + from handler.filesystem import fs_rom_handler + + validated = fs_rom_handler.validate_path( + url_cover[7:] # Remove "file://" prefix + ) + if await AnyioPath(validated).exists(): # Copy the file to the resources directory dest_path = f"{cover_file}/{size.value}.png" - await self.copy_file(Path(str(file_path)), dest_path) + await self.copy_file(validated, dest_path) if ENABLE_SCHEDULED_CONVERT_IMAGES_TO_WEBP: self.image_converter.convert_to_webp( @@ -87,14 +91,13 @@ class FSResourcesHandler(FSHandler): force=True, ) else: - log.warning(f"Cover file not found: {file_path}") + log.warning(f"Cover file not found: {str(validated)}") return None except Exception as exc: log.error(f"Unable to copy cover file {url_cover}: {str(exc)}") return None else: # Handle HTTP URLs - # Validate URL to prevent SSRF attacks validate_url_for_http_request(url_cover, "url_cover") httpx_client = ctx_httpx_client.get() @@ -103,6 +106,13 @@ class FSResourcesHandler(FSHandler): "GET", url_cover, timeout=120 ) as response: if response.status_code == status.HTTP_200_OK: + content_type = response.headers.get("content-type", "").lower() + if not content_type.startswith("image/"): + log.warning( + f"Unexpected content type for cover: {content_type}" + ) + return None + # Check if content is gzipped from response headers is_gzipped = ( response.headers.get("content-encoding", "").lower() @@ -249,21 +259,23 @@ class FSResourcesHandler(FSHandler): # Handle file:// URLs for gamelist.xml if url_screenhot.startswith("file://"): try: - file_path = AnyioPath(url_screenhot[7:]) # Remove "file://" prefix - if await file_path.exists(): + from handler.filesystem import fs_rom_handler + + validated = fs_rom_handler.validate_path( + url_screenhot[7:] # Remove "file://" prefix + ) + if await AnyioPath(validated).exists(): # Copy the file to the resources directory - await self.copy_file( - Path(str(file_path)), f"{screenshot_path}/{idx}.jpg" - ) + await self.copy_file(validated, f"{screenshot_path}/{idx}.jpg") else: - log.warning(f"Screenshot file not found: {file_path}") + log.warning(f"Screenshot file not found: {str(validated)}") return None except Exception as exc: log.error(f"Unable to copy screenshot file {url_screenhot}: {str(exc)}") return None else: # Handle HTTP URLs - # Validate URL to prevent SSRF attacks + # Validate to prevent SSRF attacks validate_url_for_http_request(url_screenhot, "url_screenshot") httpx_client = ctx_httpx_client.get() @@ -272,6 +284,13 @@ class FSResourcesHandler(FSHandler): "GET", url_screenhot, timeout=120 ) as response: if response.status_code == status.HTTP_200_OK: + content_type = response.headers.get("content-type", "").lower() + if not content_type.startswith("image/"): + log.warning( + f"Unexpected content type for screenshot: {content_type}" + ) + return None + # Check if content is gzipped from response headers is_gzipped = ( response.headers.get("content-encoding", "").lower() @@ -365,21 +384,22 @@ class FSResourcesHandler(FSHandler): # Handle file:// URLs for gamelist.xml if url_manual.startswith("file://"): try: - file_path = AnyioPath(url_manual[7:]) # Remove "file://" prefix - if await file_path.exists(): + from handler.filesystem import fs_rom_handler + + validated = fs_rom_handler.validate_path( + url_manual[7:] # Remove "file://" prefix + ) + if await AnyioPath(validated).exists(): # Copy the file to the resources directory - await self.copy_file( - Path(str(file_path)), f"{manual_path}/{rom.id}.pdf" - ) + await self.copy_file(validated, f"{manual_path}/{rom.id}.pdf") else: - log.warning(f"Manual file not found: {file_path}") + log.warning(f"Manual file not found: {str(validated)}") return None except Exception as exc: log.error(f"Unable to copy manual file {url_manual}: {str(exc)}") return None else: # Handle HTTP URL - # Validate URL to prevent SSRF attacks validate_url_for_http_request(url_manual, "url_manual") httpx_client = ctx_httpx_client.get() @@ -388,6 +408,13 @@ class FSResourcesHandler(FSHandler): "GET", url_manual, timeout=120 ) as response: if response.status_code == status.HTTP_200_OK: + content_type = response.headers.get("content-type", "").lower() + if not content_type.startswith("application/pdf"): + log.warning( + f"Unexpected content type for manual: {content_type}" + ) + return None + # Check if content is gzipped from response headers is_gzipped = ( response.headers.get("content-encoding", "").lower() @@ -440,7 +467,6 @@ class FSResourcesHandler(FSHandler): # Retroachievements async def store_ra_badge(self, url: str, path: str) -> None: - # Validate URL to prevent SSRF attacks validate_url_for_http_request(url, "url_badge") httpx_client = ctx_httpx_client.get() @@ -456,6 +482,13 @@ class FSResourcesHandler(FSHandler): try: async with httpx_client.stream("GET", url, timeout=120) as response: if response.status_code == status.HTTP_200_OK: + content_type = response.headers.get("content-type", "").lower() + if not content_type.startswith("image/"): + log.warning( + f"Unexpected content type for badge: {content_type}" + ) + return + async with await self.write_file_streamed( path=directory, filename=filename ) as f: @@ -498,7 +531,12 @@ class FSResourcesHandler(FSHandler): # Handle file:// URLs for gamelist.xml if url_media.startswith("file://"): try: - file_path = AnyioPath(url_media[7:]) # Remove "file://" prefix + from handler.filesystem import fs_rom_handler + + validated = fs_rom_handler.validate_path( + url_media[7:] # Remove "file://" prefix + ) + file_path = AnyioPath(validated) if await file_path.exists(): await self.copy_file(Path(str(file_path)), dest_path) except Exception as exc: @@ -506,7 +544,6 @@ class FSResourcesHandler(FSHandler): return None else: # Handle HTTP URLs - # Validate URL to prevent SSRF attacks validate_url_for_http_request(url_media, "url_media") httpx_client = ctx_httpx_client.get() @@ -515,6 +552,17 @@ class FSResourcesHandler(FSHandler): "GET", url_media, timeout=120 ) as response: if response.status_code == status.HTTP_200_OK: + content_type = response.headers.get("content-type", "").lower() + if not ( + content_type.startswith("image/") + or content_type.startswith("video/") + or content_type.startswith("application/pdf") + ): + log.warning( + f"Unexpected content type for media: {content_type}" + ) + return None + async with await self.write_file_streamed( path=directory, filename=filename ) as f: diff --git a/backend/handler/filesystem/roms_handler.py b/backend/handler/filesystem/roms_handler.py index 39f75f7bf..4e2079e3d 100644 --- a/backend/handler/filesystem/roms_handler.py +++ b/backend/handler/filesystem/roms_handler.py @@ -745,6 +745,6 @@ class FSRomsHandler(FSHandler): if platform_slug == UPS.PICO and fs_name.lower().endswith( PICO8_CARTRIDGE_EXTENSION ): - rom_path = self.validate_path(f"{fs_path}/{fs_name}") - return f"file://{rom_path}" + self.validate_path(f"{fs_path}/{fs_name}") + return f"file://{fs_path}/{fs_name}" return None diff --git a/backend/handler/metadata/fixtures/hltb_api_url b/backend/handler/metadata/fixtures/hltb_api_url index 67fe32dbb..4eb6b35f5 100644 --- a/backend/handler/metadata/fixtures/hltb_api_url +++ b/backend/handler/metadata/fixtures/hltb_api_url @@ -1 +1 @@ -https://howlongtobeat.com/api/finder +https://howlongtobeat.com/api/find diff --git a/backend/handler/metadata/gamelist_handler.py b/backend/handler/metadata/gamelist_handler.py index 27f7dd490..3d93a2777 100644 --- a/backend/handler/metadata/gamelist_handler.py +++ b/backend/handler/metadata/gamelist_handler.py @@ -99,10 +99,9 @@ XML_TAG_MAP: Final = { def _make_file_uri(platform_dir: str, raw_text: str) -> str: cleaned_text = raw_text.replace("./", "") - validated_path = fs_platform_handler.validate_path( - os.path.join(platform_dir, cleaned_text) - ) - return f"file://{str(validated_path)}" + joined_path = os.path.join(platform_dir, cleaned_text) + fs_platform_handler.validate_path(joined_path) + return f"file://{joined_path}" def extract_media_from_gamelist_rom( @@ -148,7 +147,9 @@ def extract_media_from_gamelist_rom( found_files = glob.glob(str(search_path)) if found_files: # trunk-ignore(mypy/literal-required) - gamelist_media[media_key] = f"file://{str(found_files[0])}" + gamelist_media[media_key] = ( + f"file://{str(Path(found_files[0]).relative_to(fs_platform_handler.base_path))}" + ) return gamelist_media diff --git a/backend/handler/metadata/hltb_handler.py b/backend/handler/metadata/hltb_handler.py index d322d464f..a3057af74 100644 --- a/backend/handler/metadata/hltb_handler.py +++ b/backend/handler/metadata/hltb_handler.py @@ -178,9 +178,11 @@ class HLTBHandler(MetadataHandler): self.base_url = "https://howlongtobeat.com" self.user_endpoint = f"{self.base_url}/api/user" self.stats_endpoint = f"{self.base_url}/api/stats/games?platform=1&year=2000" - self.search_url = f"{self.base_url}/api/search" + self.search_url = f"{self.base_url}/api/find" self.search_init_url = f"{self.search_url}/init" self.security_token = None + self.hp_key = None + self.hp_val = None self.min_similarity_score: Final = 0.85 # HLTB rotates their search endpoint regularly @@ -226,7 +228,10 @@ class HLTBHandler(MetadataHandler): timeout=10, ) response.raise_for_status() - self.security_token = response.json().get("token", None) + data = response.json() + self.security_token = data.get("token", None) + self.hp_key = data.get("hpKey", None) + self.hp_val = data.get("hpVal", None) except Exception as e: log.warning("Unexpected error fetching HLTB security token: %s", e) @@ -262,9 +267,12 @@ class HLTBHandler(MetadataHandler): "Content-Type": "application/json", "Referer": "https://howlongtobeat.com", "User-Agent": f"RomM/{get_version()}", - "X-Auth-Token": self.security_token, + "x-auth-token": self.security_token, + "x-hp-key": self.hp_key, + "x-hp-val": self.hp_val, } + payload[self.hp_key] = self.hp_val log.debug( "HowLongToBeat API request: URL=%s, Headers=%s, Payload=%s, Timeout=%s", url, diff --git a/backend/tests/config/test_config_loader.py b/backend/tests/config/test_config_loader.py index c1792858a..c3d2e1cca 100644 --- a/backend/tests/config/test_config_loader.py +++ b/backend/tests/config/test_config_loader.py @@ -14,12 +14,25 @@ def test_config_loader(): os.path.join(Path(__file__).resolve().parent, "fixtures", "config/config.yml") ) - assert loader.config.EXCLUDED_PLATFORMS == ["romm"] - assert loader.config.EXCLUDED_SINGLE_EXT == ["xml"] - assert loader.config.EXCLUDED_SINGLE_FILES == ["info.txt"] - assert loader.config.EXCLUDED_MULTI_FILES == ["my_multi_file_game", "DLC"] - assert loader.config.EXCLUDED_MULTI_PARTS_EXT == ["txt"] - assert loader.config.EXCLUDED_MULTI_PARTS_FILES == ["data.xml"] + assert loader.config.EXCLUDED_PLATFORMS == {*DEFAULT_EXCLUDED_DIRS, "romm"} + assert loader.config.EXCLUDED_SINGLE_EXT == { + *(e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS), + "xml", + } + assert loader.config.EXCLUDED_SINGLE_FILES == {*DEFAULT_EXCLUDED_FILES, "info.txt"} + assert loader.config.EXCLUDED_MULTI_FILES == { + *DEFAULT_EXCLUDED_DIRS, + "my_multi_file_game", + "DLC", + } + assert loader.config.EXCLUDED_MULTI_PARTS_EXT == { + *(e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS), + "txt", + } + assert loader.config.EXCLUDED_MULTI_PARTS_FILES == { + *DEFAULT_EXCLUDED_FILES, + "data.xml", + } assert loader.config.PLATFORMS_BINDING == {"gc": "ngc"} assert loader.config.PLATFORMS_VERSIONS == {"naomi": "arcade"} assert loader.config.ROMS_FOLDER_NAME == "ROMS" @@ -63,16 +76,16 @@ def test_empty_config_loader(): ) ) - assert loader.config.EXCLUDED_PLATFORMS == DEFAULT_EXCLUDED_DIRS - assert loader.config.EXCLUDED_SINGLE_EXT == [ + assert loader.config.EXCLUDED_PLATFORMS == set(DEFAULT_EXCLUDED_DIRS) + assert loader.config.EXCLUDED_SINGLE_EXT == { e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS - ] - assert loader.config.EXCLUDED_SINGLE_FILES == DEFAULT_EXCLUDED_FILES - assert loader.config.EXCLUDED_MULTI_FILES == DEFAULT_EXCLUDED_DIRS - assert loader.config.EXCLUDED_MULTI_PARTS_EXT == [ + } + assert loader.config.EXCLUDED_SINGLE_FILES == set(DEFAULT_EXCLUDED_FILES) + assert loader.config.EXCLUDED_MULTI_FILES == set(DEFAULT_EXCLUDED_DIRS) + assert loader.config.EXCLUDED_MULTI_PARTS_EXT == { e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS - ] - assert loader.config.EXCLUDED_MULTI_PARTS_FILES == DEFAULT_EXCLUDED_FILES + } + assert loader.config.EXCLUDED_MULTI_PARTS_FILES == set(DEFAULT_EXCLUDED_FILES) assert loader.config.PLATFORMS_BINDING == {} assert loader.config.PLATFORMS_VERSIONS == {} assert loader.config.ROMS_FOLDER_NAME == "roms" diff --git a/backend/tests/endpoints/sockets/test_scan.py b/backend/tests/endpoints/sockets/test_scan.py index c551548f2..df520bffc 100644 --- a/backend/tests/endpoints/sockets/test_scan.py +++ b/backend/tests/endpoints/sockets/test_scan.py @@ -263,7 +263,7 @@ class TestGetPico8CoverUrl: fs_name="mygame.p8.png", fs_path="pico/roms", ) - expected = f"file://{Path(LIBRARY_BASE_PATH).resolve() / 'pico/roms' / 'mygame.p8.png'}" + expected = f"file://pico/roms/mygame.p8.png" assert url == expected def test_returns_none_for_non_pico8_platform(self, handler: FSRomsHandler): diff --git a/backend/tests/endpoints/test_config.py b/backend/tests/endpoints/test_config.py index 76599031f..91c64ab52 100644 --- a/backend/tests/endpoints/test_config.py +++ b/backend/tests/endpoints/test_config.py @@ -24,16 +24,16 @@ def test_config(client): assert response.status_code == status.HTTP_200_OK config = response.json() - assert config.get("EXCLUDED_PLATFORMS") == DEFAULT_EXCLUDED_DIRS - assert config.get("EXCLUDED_SINGLE_EXT") == [ + assert config.get("EXCLUDED_PLATFORMS") == sorted(DEFAULT_EXCLUDED_DIRS) + assert config.get("EXCLUDED_SINGLE_EXT") == sorted( e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS - ] - assert config.get("EXCLUDED_SINGLE_FILES") == DEFAULT_EXCLUDED_FILES - assert config.get("EXCLUDED_MULTI_FILES") == DEFAULT_EXCLUDED_DIRS - assert config.get("EXCLUDED_MULTI_PARTS_EXT") == [ + ) + assert config.get("EXCLUDED_SINGLE_FILES") == sorted(DEFAULT_EXCLUDED_FILES) + assert config.get("EXCLUDED_MULTI_FILES") == sorted(DEFAULT_EXCLUDED_DIRS) + assert config.get("EXCLUDED_MULTI_PARTS_EXT") == sorted( e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS - ] - assert config.get("EXCLUDED_MULTI_PARTS_FILES") == DEFAULT_EXCLUDED_FILES + ) + assert config.get("EXCLUDED_MULTI_PARTS_FILES") == sorted(DEFAULT_EXCLUDED_FILES) assert config.get("PLATFORMS_BINDING") == {} assert not config.get("SKIP_HASH_CALCULATION") diff --git a/backend/tests/handler/filesystem/test_base_handler.py b/backend/tests/handler/filesystem/test_base_handler.py index 16eb3ad48..7329c16cb 100644 --- a/backend/tests/handler/filesystem/test_base_handler.py +++ b/backend/tests/handler/filesystem/test_base_handler.py @@ -151,8 +151,8 @@ class TestFSHandler: # Mock configuration with patch("handler.filesystem.base_handler.cm.get_config") as mock_config: - mock_config.return_value.EXCLUDED_SINGLE_EXT = ["tmp"] - mock_config.return_value.EXCLUDED_SINGLE_FILES = ["test.txt"] + mock_config.return_value.EXCLUDED_SINGLE_EXT = {"tmp"} + mock_config.return_value.EXCLUDED_SINGLE_FILES = {"test.txt"} result = handler.exclude_single_files(files) diff --git a/backend/tests/handler/filesystem/test_firmware_handler.py b/backend/tests/handler/filesystem/test_firmware_handler.py index ab6981840..556fedf5b 100644 --- a/backend/tests/handler/filesystem/test_firmware_handler.py +++ b/backend/tests/handler/filesystem/test_firmware_handler.py @@ -22,12 +22,12 @@ class TestFSFirmwareHandler: @pytest.fixture def config(self): return Config( - EXCLUDED_PLATFORMS=[], - EXCLUDED_SINGLE_EXT=["tmp"], - EXCLUDED_SINGLE_FILES=[], - EXCLUDED_MULTI_FILES=[], - EXCLUDED_MULTI_PARTS_EXT=[], - EXCLUDED_MULTI_PARTS_FILES=[], + EXCLUDED_PLATFORMS=set(), + EXCLUDED_SINGLE_EXT={"tmp"}, + EXCLUDED_SINGLE_FILES=set(), + EXCLUDED_MULTI_FILES=set(), + EXCLUDED_MULTI_PARTS_EXT=set(), + EXCLUDED_MULTI_PARTS_FILES=set(), PLATFORMS_BINDING={}, PLATFORMS_VERSIONS={}, ROMS_FOLDER_NAME="roms", diff --git a/backend/tests/handler/filesystem/test_platforms_handler.py b/backend/tests/handler/filesystem/test_platforms_handler.py index 091305a06..2cbf7118f 100644 --- a/backend/tests/handler/filesystem/test_platforms_handler.py +++ b/backend/tests/handler/filesystem/test_platforms_handler.py @@ -17,12 +17,12 @@ class TestFSPlatformsHandler: @pytest.fixture def config(self): return Config( - EXCLUDED_PLATFORMS=["romm", "excluded_platform"], - EXCLUDED_SINGLE_EXT=["tmp"], - EXCLUDED_SINGLE_FILES=[], - EXCLUDED_MULTI_FILES=[], - EXCLUDED_MULTI_PARTS_EXT=[], - EXCLUDED_MULTI_PARTS_FILES=[], + EXCLUDED_PLATFORMS={"romm", "excluded_platform"}, + EXCLUDED_SINGLE_EXT={"tmp"}, + EXCLUDED_SINGLE_FILES=set(), + EXCLUDED_MULTI_FILES=set(), + EXCLUDED_MULTI_PARTS_EXT=set(), + EXCLUDED_MULTI_PARTS_FILES=set(), PLATFORMS_BINDING={}, PLATFORMS_VERSIONS={}, ROMS_FOLDER_NAME="roms", @@ -32,12 +32,12 @@ class TestFSPlatformsHandler: @pytest.fixture def config_custom_folder(self): return Config( - EXCLUDED_PLATFORMS=[], - EXCLUDED_SINGLE_EXT=[], - EXCLUDED_SINGLE_FILES=[], - EXCLUDED_MULTI_FILES=[], - EXCLUDED_MULTI_PARTS_EXT=[], - EXCLUDED_MULTI_PARTS_FILES=[], + EXCLUDED_PLATFORMS=set(), + EXCLUDED_SINGLE_EXT=set(), + EXCLUDED_SINGLE_FILES=set(), + EXCLUDED_MULTI_FILES=set(), + EXCLUDED_MULTI_PARTS_EXT=set(), + EXCLUDED_MULTI_PARTS_FILES=set(), PLATFORMS_BINDING={}, PLATFORMS_VERSIONS={}, ROMS_FOLDER_NAME="ROMS", @@ -194,7 +194,7 @@ class TestFSPlatformsHandler: with patch( "handler.filesystem.platforms_handler.cm.get_config", return_value=config ): - config.EXCLUDED_PLATFORMS = ["psx"] + config.EXCLUDED_PLATFORMS = {"psx"} result = await handler.get_platforms() assert "n64" in result diff --git a/backend/tests/handler/filesystem/test_roms_handler.py b/backend/tests/handler/filesystem/test_roms_handler.py index 4694d64bc..c93ca38c9 100644 --- a/backend/tests/handler/filesystem/test_roms_handler.py +++ b/backend/tests/handler/filesystem/test_roms_handler.py @@ -26,12 +26,12 @@ class TestFSRomsHandler: @pytest.fixture def config(self): return Config( - EXCLUDED_PLATFORMS=[], - EXCLUDED_SINGLE_EXT=["tmp"], - EXCLUDED_SINGLE_FILES=["excluded_test.tmp"], - EXCLUDED_MULTI_FILES=["excluded_multi"], - EXCLUDED_MULTI_PARTS_EXT=["tmp"], - EXCLUDED_MULTI_PARTS_FILES=["excluded_part.bin"], + EXCLUDED_PLATFORMS=set(), + EXCLUDED_SINGLE_EXT={"tmp"}, + EXCLUDED_SINGLE_FILES={"excluded_test.tmp"}, + EXCLUDED_MULTI_FILES={"excluded_multi"}, + EXCLUDED_MULTI_PARTS_EXT={"tmp"}, + EXCLUDED_MULTI_PARTS_FILES={"excluded_part.bin"}, PLATFORMS_BINDING={}, PLATFORMS_VERSIONS={}, ROMS_FOLDER_NAME="roms", @@ -107,12 +107,12 @@ class TestFSRomsHandler: m.setattr( "handler.filesystem.roms_handler.cm.get_config", lambda: Config( - EXCLUDED_PLATFORMS=[], - EXCLUDED_SINGLE_EXT=[], - EXCLUDED_SINGLE_FILES=[], - EXCLUDED_MULTI_FILES=[], - EXCLUDED_MULTI_PARTS_EXT=[], - EXCLUDED_MULTI_PARTS_FILES=[], + EXCLUDED_PLATFORMS=set(), + EXCLUDED_SINGLE_EXT=set(), + EXCLUDED_SINGLE_FILES=set(), + EXCLUDED_MULTI_FILES=set(), + EXCLUDED_MULTI_PARTS_EXT=set(), + EXCLUDED_MULTI_PARTS_FILES=set(), PLATFORMS_BINDING={}, PLATFORMS_VERSIONS={}, ROMS_FOLDER_NAME="roms", @@ -134,12 +134,12 @@ class TestFSRomsHandler: m.setattr( "handler.filesystem.roms_handler.cm.get_config", lambda: Config( - EXCLUDED_PLATFORMS=[], - EXCLUDED_SINGLE_EXT=[], - EXCLUDED_SINGLE_FILES=[], - EXCLUDED_MULTI_FILES=[], - EXCLUDED_MULTI_PARTS_EXT=[], - EXCLUDED_MULTI_PARTS_FILES=[], + EXCLUDED_PLATFORMS=set(), + EXCLUDED_SINGLE_EXT=set(), + EXCLUDED_SINGLE_FILES=set(), + EXCLUDED_MULTI_FILES=set(), + EXCLUDED_MULTI_PARTS_EXT=set(), + EXCLUDED_MULTI_PARTS_FILES=set(), PLATFORMS_BINDING={}, PLATFORMS_VERSIONS={}, ROMS_FOLDER_NAME="roms", @@ -237,12 +237,12 @@ class TestFSRomsHandler: """Test exclude_multi_roms with no exclusions""" roms = ["Game1", "Game2", "Game3"] config = Config( - EXCLUDED_PLATFORMS=[], - EXCLUDED_SINGLE_EXT=[], - EXCLUDED_SINGLE_FILES=[], - EXCLUDED_MULTI_FILES=[], - EXCLUDED_MULTI_PARTS_EXT=[], - EXCLUDED_MULTI_PARTS_FILES=[], + EXCLUDED_PLATFORMS=set(), + EXCLUDED_SINGLE_EXT=set(), + EXCLUDED_SINGLE_FILES=set(), + EXCLUDED_MULTI_FILES=set(), + EXCLUDED_MULTI_PARTS_EXT=set(), + EXCLUDED_MULTI_PARTS_FILES=set(), PLATFORMS_BINDING={}, PLATFORMS_VERSIONS={}, ROMS_FOLDER_NAME="roms",