"""ModDB page parsing and download selection."""
from __future__ import annotations
import html
import dataclasses
import logging
import re
import urllib.error
import urllib.parse
import urllib.request
from collections import OrderedDict
from typing import Optional
from doomdeck.domain.downloads import DownloadPolicy
from doomdeck.domain.models import DoomDeckError, ModDBDownload
from doomdeck.infrastructure.downloads import DEFAULT_USER_AGENT
MODDB_BASE_URL = "https://www.moddb.com"
BRUTAL_DOOM_MODDB_URL = "https://www.moddb.com/mods/brutal-doom"
BRUTAL_DOOM_DOWNLOADS_URL = "https://www.moddb.com/mods/brutal-doom/downloads"
def fetch_text_url(
url: str,
logger: Optional[logging.Logger] = None,
allowed_hosts: Optional[set[str]] = None,
user_agent: str = DEFAULT_USER_AGENT,
) -> str:
DownloadPolicy.for_hosts(allowed_hosts).validate_url(url)
if logger:
logger.debug("Fetch metadata page: %s", url)
request = urllib.request.Request(
url,
headers={
"User-Agent": user_agent,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
},
)
# URL policy is validated before opening the request.
with urllib.request.urlopen(request, timeout=30) as response: # nosec B310
content_type = response.headers.get_content_charset() or "utf-8"
return response.read().decode(content_type, errors="replace")
def safe_download_name(value: str, fallback: str) -> str:
name = value.rstrip("/").split("/")[-1].split("?")[0]
name = re.sub(r"[^A-Za-z0-9._-]+", "-", name).strip(".-")
return name or fallback
def strip_html(text: str) -> str:
text = re.sub(r"(?is)", " ", text)
text = re.sub(r"(?is)