Something went wrong. Try again.
Steam Deck Doom modding setup automation for Doom Runner, UZDoom, Brutal Doom, and Project Brutality.
brutal-doom moddb python doom-runner linux-gaming project-brutality uzdoom doom steam-deck
Something went wrong. Try again.
12 kB · 295 lines
Python
at master
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296"""ModDB page parsing and download selection."""from __future__ import annotations
import htmlimport dataclassesimport loggingimport reimport urllib.errorimport urllib.parseimport urllib.requestfrom collections import OrderedDictfrom typing import Optional
from doomdeck.domain.downloads import DownloadPolicyfrom doomdeck.domain.models import DoomDeckError, ModDBDownloadfrom doomdeck.infrastructure.downloads import DEFAULT_USER_AGENT
MODDB_BASE_URL = "https://www.moddb.com"BRUTAL_DOOM_MODDB_URL = "https://www.moddb.com/mods/brutal-doom"BRUTAL_DOOM_DOWNLOADS_URL = "https://www.moddb.com/mods/brutal-doom/downloads"
def fetch_text_url( url: str, logger: Optional[logging.Logger] = None, allowed_hosts: Optional[set[str]] = None, user_agent: str = DEFAULT_USER_AGENT,) -> str: DownloadPolicy.for_hosts(allowed_hosts).validate_url(url) if logger: logger.debug("Fetch metadata page: %s", url) request = urllib.request.Request( url, headers={ "User-Agent": user_agent, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", }, ) # URL policy is validated before opening the request. with urllib.request.urlopen(request, timeout=30) as response: # nosec B310 content_type = response.headers.get_content_charset() or "utf-8" return response.read().decode(content_type, errors="replace")
def safe_download_name(value: str, fallback: str) -> str: name = value.rstrip("/").split("/")[-1].split("?")[0] name = re.sub(r"[^A-Za-z0-9._-]+", "-", name).strip(".-") return name or fallback
def strip_html(text: str) -> str: text = re.sub(r"(?is)<script\b.*?</script>", " ", text) text = re.sub(r"(?is)<style\b.*?</style>", " ", text) text = re.sub(r"(?is)<[^>]+>", " ", text) text = html.unescape(text) return re.sub(r"\s+", " ", text).strip()
def html_text_lines(text: str) -> list[str]: text = re.sub(r"(?i)<br\s*/?>", "\n", text) text = re.sub(r"(?i)</(p|div|li|h[1-6]|dt|dd|tr|td|th)>", "\n", text) text = re.sub(r"(?is)<script\b.*?</script>", " ", text) text = re.sub(r"(?is)<style\b.*?</style>", " ", text) text = re.sub(r"(?is)<[^>]+>", " ", text) lines = [re.sub(r"\s+", " ", html.unescape(line)).strip() for line in text.splitlines()] return [line for line in lines if line]
def html_links(text: str) -> list[tuple[str, str]]: links: list[tuple[str, str]] = [] for match in re.finditer(r"""(?is)<a\b[^>]*\bhref\s*=\s*["'](?P<href>[^"']+)["'][^>]*>(?P<body>.*?)</a>""", text): href = html.unescape(match.group("href")).strip() body = strip_html(match.group("body")) if href: links.append((href, body)) return links
def absolute_moddb_url(href: str, base: str = MODDB_BASE_URL) -> str: return urllib.parse.urljoin(base, html.unescape(href).strip())
def line_after_label(lines: list[str], label: str) -> str: label_lower = label.lower() for idx, line in enumerate(lines): normalized = line.rstrip(":").strip().lower() if normalized != label_lower: continue for candidate in lines[idx + 1 : idx + 4]: if candidate.rstrip(":").strip().lower() != label_lower: return candidate.strip() return ""
def extract_moddb_download_link(page_html: str, page_url: str) -> str: for href, text in html_links(page_html): if re.search(r"/(?:downloads|addons)/start/", href) and "download" in text.lower(): return absolute_moddb_url(href, page_url) for href, _text in html_links(page_html): if re.search(r"/(?:downloads|addons)/start/", href): return absolute_moddb_url(href, page_url) match = re.search(r"""(?i)href\s*=\s*["'](?P<href>[^"']*/(?:downloads|addons)/start/\d+[^"']*)["']""", page_html) if match: return absolute_moddb_url(match.group("href"), page_url) raise DoomDeckError(f"Could not find a ModDB start-download link on {page_url}")
def resolve_moddb_download_url(start_url: str, page_url: str, logger: logging.Logger, user_agent: str = DEFAULT_USER_AGENT) -> str: def first_mirror(html_text: str, base_url: str) -> Optional[str]: for href, _text in html_links(html_text): if re.search(r"/(?:downloads|addons)/mirror/", href): return absolute_moddb_url(href, base_url) return None
try: start_html = fetch_text_url(start_url, logger, user_agent=user_agent) except urllib.error.URLError as exc: raise DoomDeckError(f"Could not read ModDB download page {start_url}: {exc}") from exc
mirror = first_mirror(start_html, start_url) if mirror: return mirror
mirrors_url = start_url.rstrip("/") + "/all" try: mirrors_html = fetch_text_url(mirrors_url, logger, user_agent=user_agent) except urllib.error.URLError as exc: logger.warning("Could not read ModDB mirror list %s: %s", mirrors_url, exc) return start_url mirror = first_mirror(mirrors_html, mirrors_url) if mirror: return mirror logger.warning("No explicit ModDB mirror found; using start URL directly: %s", start_url) return start_url
def extract_brutal_doom_pages(page_html: str, base_url: str) -> list[tuple[str, str]]: pages: OrderedDict[str, str] = OrderedDict()
def title_quality(value: str) -> int: lower = value.lower() if "brutal doom" in lower: return 100 if re.search(r"\bbd\s*v?\d+", lower): return 80 if "comments" in lower or lower.startswith("image:"): return 0 return 10 if value else 0
for href, text in html_links(page_html): url = absolute_moddb_url(href, base_url) parsed = urllib.parse.urlparse(url) if not parsed.path.startswith("/mods/brutal-doom/downloads/"): continue title = text.strip() if not title: title = parsed.path.rstrip("/").split("/")[-1].replace("-", " ").title() if url not in pages or title_quality(title) > title_quality(pages[url]): pages[url] = title return [(title, url) for url, title in pages.items()]
@dataclasses.dataclass(frozen=True)class ModDBPageCandidate: title: str url: str index: int
@dataclasses.dataclass(frozen=True)class BrutalDoomSelectionPolicy: channel: str
def score(self, candidate: ModDBPageCandidate) -> int: slug = urllib.parse.urlparse(candidate.url).path.rstrip("/").split("/")[-1] lower = f"{candidate.title} {slug}".lower() title_compact = re.sub(r"[^a-z0-9]+", "", candidate.title.lower()) slug_compact = re.sub(r"[^a-z0-9]+", "", slug.lower()) if "brutaldoom" not in title_compact and "brutaldoomv" not in slug_compact and not re.search(r"\bbd\s*v?\d+", candidate.title.lower()): return -1000 rejected = [ "platinum", "kickass", "monsters only", "monster only", "metal soundtrack", "soundtrack", "eday", "extermination day", "bolognese", "meatgrinder", "black edition", "addon", "launcher", ] if any(token in lower for token in rejected): return -1000 score = 100 - candidate.index versions: list[int] = [] for value in re.findall(r"\bv\s*(\d+)", lower): # ModDB slugs sometimes collapse dots, e.g. v21.50.0 -> v21500. major_text = value[:2] if len(value) >= 4 else value versions.append(int(major_text)) if versions: score += max(versions) * 10 is_test = any(token in lower for token in ["beta", "test", "demo"]) if self.channel == "stable": score += 100 if not is_test else -200 else: score += 30 if is_test else 0 if "full version" in lower: score += 20 return score
def brutal_doom_candidate_score(title: str, url: str, channel: str, index: int) -> int: return BrutalDoomSelectionPolicy(channel).score(ModDBPageCandidate(title, url, index))
def select_brutal_doom_download(channel: str, logger: logging.Logger, user_agent: str = DEFAULT_USER_AGENT) -> ModDBDownload: pages: list[tuple[str, str]] = [] errors: list[str] = [] for source_url in [BRUTAL_DOOM_MODDB_URL, BRUTAL_DOOM_DOWNLOADS_URL]: try: pages.extend(extract_brutal_doom_pages(fetch_text_url(source_url, logger, user_agent=user_agent), source_url)) except urllib.error.URLError as exc: errors.append(f"{source_url}: {exc}")
seen: set[str] = set() unique_pages: list[tuple[str, str]] = [] for title, url in pages: if url in seen: continue seen.add(url) unique_pages.append((title, url)) if not unique_pages: detail = "; ".join(errors) if errors else "no matching download links found" raise DoomDeckError(f"Could not discover Brutal Doom downloads on ModDB: {detail}")
policy = BrutalDoomSelectionPolicy(channel) candidates = [ ModDBPageCandidate(title=title, url=url, index=index) for index, (title, url) in enumerate(unique_pages) ] ranked = sorted(candidates, key=policy.score, reverse=True) for candidate in ranked: if policy.score(candidate) <= 0: continue logger.info("Selected Brutal Doom ModDB page: %s", candidate.title) try: page_html = fetch_text_url(candidate.url, logger, user_agent=user_agent) except urllib.error.URLError as exc: logger.warning("Could not read Brutal Doom page %s: %s", candidate.url, exc) continue lines = html_text_lines(page_html) filename = line_after_label(lines, "Filename") or safe_download_name(candidate.url, "brutal-doom.zip") updated = line_after_label(lines, "Updated") md5 = line_after_label(lines, "MD5 Hash") start_url = extract_moddb_download_link(page_html, candidate.url) return ModDBDownload( title=candidate.title, page_url=candidate.url, filename=safe_download_name(filename, "brutal-doom.zip"), download_url=resolve_moddb_download_url(start_url, candidate.url, logger, user_agent=user_agent), updated=updated, md5=md5, )
raise DoomDeckError(f"Could not find a suitable Brutal Doom ModDB download for channel '{channel}'")
def select_moddb_wad_download(page_url: str, logger: logging.Logger, user_agent: str = DEFAULT_USER_AGENT) -> ModDBDownload: parsed = urllib.parse.urlparse(page_url) if parsed.scheme not in {"http", "https"} or not parsed.netloc.endswith("moddb.com"): raise DoomDeckError(f"ModDB WAD URL must be a ModDB URL, got: {page_url}")
try: page_html = fetch_text_url(page_url, logger, user_agent=user_agent) except urllib.error.URLError as exc: raise DoomDeckError(f"Could not read ModDB page {page_url}: {exc}") from exc
lines = html_text_lines(page_html) title = next((line for line in lines if line.lower() not in {"hello guest", "description"}), page_url) filename = line_after_label(lines, "Filename") or safe_download_name(page_url, "moddb-wad.zip") updated = line_after_label(lines, "Updated") md5 = line_after_label(lines, "MD5 Hash") start_url = extract_moddb_download_link(page_html, page_url) return ModDBDownload( title=title, page_url=page_url, filename=safe_download_name(filename, "moddb-wad.zip"), download_url=resolve_moddb_download_url(start_url, page_url, logger, user_agent=user_agent), updated=updated, md5=md5, )