diff --git a/.dockerignore b/.dockerignore new file mode 100644 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,26 @@ +# Python-generated files +__pycache__/ +*.py[oc] +build/ +dist/ +wheels/ +*.egg-info +.pytest_cache/ +.venv/ +.mypy_cache/ +.ruff_cache/ +.coverage +htmlcov/ + +# Random junk +.env +.env.* +.DS_Store +data/ +testdata/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo diff --git a/.gitignore b/.gitignore new file mode 100644 --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +# Python-generated files +__pycache__/ +*.py[oc] +build/ +dist/ +wheels/ +*.egg-info +.pytest_cache/ +.venv/ +.mypy_cache/ +.ruff_cache/ +.coverage +htmlcov/ + +# Random junk +.env +.env.* +.DS_Store +data/ +testdata/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo diff --git a/.python-version b/.python-version new file mode 100644 --- /dev/null +++ b/.python-version @@ -0,0 +1,1 @@ +3.12 diff --git a/.tangled/workflows/run-tests.yml b/.tangled/workflows/run-tests.yml new file mode 100644 --- /dev/null +++ b/.tangled/workflows/run-tests.yml @@ -0,0 +1,18 @@ +when: + - event: ["push", "manual"] + branch: ["next"] + +engine: nixery + +dependencies: + nixpkgs: + - uv + - ruff + - python312 + +steps: + - name: run tests + command: | + uv run --python python3.12 pytest -vv + uv run --python python3.12 ruff check . + uv run --python python3.12 mypy . diff --git a/Containerfile b/Containerfile new file mode 100644 --- /dev/null +++ b/Containerfile @@ -0,0 +1,41 @@ +FROM python:3.12-alpine +COPY --from=ghcr.io/astral-sh/uv:0.7.12 /uv /uvx /bin/ + +# Install build tools & runtime dependencies +RUN apk add --no-cache \ + ffmpeg \ + file \ + libmagic + +RUN mkdir -p /app/data +WORKDIR /app + +# switch to a non-root user +RUN adduser -D -u 1000 app && \ + chown -R app:app /app +USER app + +# Enable bytecode compilation +ENV UV_COMPILE_BYTECODE=1 + +# Copy from the cache instead of linking since it's a mounted volume +ENV UV_LINK_MODE=copy + +# Install the project's dependencies using the lockfile and settings +COPY ./uv.lock ./pyproject.toml /app/ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --locked --no-install-project --no-dev + +# Define app data volume +VOLUME /app/data + +# Then, add the rest of the project source code and install it +COPY . /app +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --locked --no-dev + +# Place executables in the environment at the front of the path +ENV PATH="/app/.venv/bin:$PATH" + +# Set entrypoint to run the app using uv +ENTRYPOINT ["uv", "run", "main.py"] diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +# XPost (next) + +> [!NOTE] +> For xpost v1, see the master branch + +Xpost is a social media cross-posting tool that differs from others by using streaming APIs to allow instant, zero-input cross-posting. This means you can continue posting on your preferred platform without using special apps. + +See [docs](./docs/README.md) for more info and configuration options! diff --git a/atproto/__init__.py b/atproto/__init__.py new file mode 100644 --- /dev/null +++ b/atproto/__init__.py diff --git a/atproto/models.py b/atproto/models.py new file mode 100644 --- /dev/null +++ b/atproto/models.py @@ -0,0 +1,294 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Any + + +URI = "at://" +URI_LEN = len(URI) + + +class AtUri: + @classmethod + def record_uri(cls, uri: str) -> tuple[str, str, str]: + if not uri.startswith(URI): + raise ValueError(f"Ivalid record uri {uri}!") + + did, collection, rid = uri[URI_LEN:].split("/") + if not (did and collection and rid): + raise ValueError(f"Ivalid record uri {uri}!") + + return did, collection, rid + + +@dataclass(kw_only=True) +class StrongRef: + uri: str + cid: str + + def to_dict(self) -> dict[str, Any]: + return {"uri": self.uri, "cid": self.cid} + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "StrongRef": + return cls(uri=data["uri"], cid=data["cid"]) + + +@dataclass(kw_only=True) +class ReplyRef: + root: StrongRef + parent: StrongRef + + def to_dict(self) -> dict[str, Any]: + return { + "root": self.root.to_dict(), + "parent": self.parent.to_dict(), + } + + +@dataclass(kw_only=True) +class FacetIndex: + byte_start: int + byte_end: int + + def to_dict(self) -> dict[str, int]: + return {"byteStart": self.byte_start, "byteEnd": self.byte_end} + + +@dataclass(kw_only=True) +class FacetFeature(ABC): + @abstractmethod + def to_dict(self) -> dict[str, Any]: + pass + + +@dataclass(kw_only=True) +class MentionFeature(FacetFeature): + did: str + + def to_dict(self) -> dict[str, Any]: + return { + "$type": "app.bsky.richtext.facet#mention", + "did": self.did, + } + + +@dataclass(kw_only=True) +class LinkFeature(FacetFeature): + uri: str + + def to_dict(self) -> dict[str, Any]: + return { + "$type": "app.bsky.richtext.facet#link", + "uri": self.uri, + } + + +@dataclass(kw_only=True) +class TagFeature(FacetFeature): + tag: str + + def to_dict(self) -> dict[str, Any]: + return { + "$type": "app.bsky.richtext.facet#tag", + "tag": self.tag, + } + + +@dataclass(kw_only=True) +class Facet: + index: FacetIndex + features: list[FacetFeature] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "index": self.index.to_dict(), + "features": [f.to_dict() for f in self.features], + } + + +@dataclass(kw_only=True) +class ImageEmbed: + image: bytes + alt: str | None = None + aspect_ratio: tuple[int, int] | None = None + + def to_dict(self, blob_ref: dict[str, Any]) -> dict[str, Any]: + data: dict[str, Any] = { + "image": blob_ref, + "alt": self.alt or "", + } + if self.aspect_ratio: + data["aspectRatio"] = { + "width": self.aspect_ratio[0], + "height": self.aspect_ratio[1], + } + return data + + +@dataclass(kw_only=True) +class VideoEmbed: + video: bytes + alt: str | None = None + aspect_ratio: tuple[int, int] | None = None + + def to_dict(self, blob_ref: dict[str, Any]) -> dict[str, Any]: + data: dict[str, Any] = { + "$type": "app.bsky.embed.video", + "video": blob_ref, + } + if self.alt: + data["alt"] = self.alt + if self.aspect_ratio: + data["aspectRatio"] = { + "width": self.aspect_ratio[0], + "height": self.aspect_ratio[1], + } + return data + + +@dataclass(kw_only=True) +class RecordEmbed: + record: StrongRef + + def to_dict(self) -> dict[str, Any]: + return { + "$type": "app.bsky.embed.record", + "record": self.record.to_dict(), + } + + +@dataclass(kw_only=True) +class RecordWithMediaEmbed: + record: StrongRef + media: ImageEmbed | VideoEmbed + media_blob_ref: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + media_data = self.media.to_dict(self.media_blob_ref) + media_type = ( + "app.bsky.embed.images" + if isinstance(self.media, ImageEmbed) + else "app.bsky.embed.video" + ) + return { + "$type": "app.bsky.embed.recordWithMedia", + "record": self.record.to_dict(), + "media": { + "$type": media_type, + **media_data, + }, + } + + +@dataclass(kw_only=True) +class SelfLabel: + val: str + + def to_dict(self) -> dict[str, str]: + return {"val": self.val} + + +@dataclass(kw_only=True) +class SelfLabels: + values: list[SelfLabel] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "$type": "com.atproto.label.defs#selfLabels", + "values": [v.to_dict() for v in self.values], + } + + +@dataclass(kw_only=True) +class PostRecord: + text: str + created_at: str + facets: list[Facet] | None = None + embed: dict[str, Any] | None = None + reply: ReplyRef | None = None + langs: list[str] | None = None + labels: SelfLabels | None = None + + def to_dict(self) -> dict[str, Any]: + data: dict[str, Any] = { + "$type": "app.bsky.feed.post", + "text": self.text, + "createdAt": self.created_at, + } + if self.facets: + data["facets"] = [f.to_dict() for f in self.facets] + if self.embed: + data["embed"] = self.embed + if self.reply: + data["reply"] = self.reply.to_dict() + if self.langs: + data["langs"] = self.langs + if self.labels: + data["labels"] = self.labels.to_dict() + return data + + +@dataclass(kw_only=True) +class CreateRecordResponse: + uri: str + cid: str + commit: dict[str, Any] | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "CreateRecordResponse": + return cls( + uri=data.get("uri", ""), + cid=data.get("cid", ""), + commit=data.get("commit"), + ) + + +@dataclass(kw_only=True) +class RepostRecord: + subject: StrongRef + created_at: str + + def to_dict(self) -> dict[str, Any]: + data: dict[str, Any] = { + "$type": "app.bsky.feed.repost", + "createdAt": self.created_at, + "subject": self.subject.to_dict(), + } + return data + + +@dataclass(kw_only=True) +class ThreadGate: + post: str = "" + created_at: str + allow: list[dict[str, Any]] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + data: dict[str, Any] = { + "$type": "app.bsky.feed.threadgate", + "post": self.post, + "createdAt": self.created_at, + } + if self.allow is not None: + data["allow"] = self.allow + return data + + +@dataclass(kw_only=True) +class PostGate: + post: str + created_at: str + detached_embedding_uris: list[str] | None = None + embedding_rules: list[dict[str, Any]] | None = None + + def to_dict(self) -> dict[str, Any]: + data: dict[str, Any] = { + "$type": "app.bsky.feed.postgate", + "post": self.post, + "createdAt": self.created_at, + } + if self.detached_embedding_uris is not None: + data["detachedEmbeddingUris"] = self.detached_embedding_uris + if self.embedding_rules is not None: + data["embeddingRules"] = self.embedding_rules + return data diff --git a/atproto/store.py b/atproto/store.py new file mode 100644 --- /dev/null +++ b/atproto/store.py @@ -0,0 +1,235 @@ +import base64 +import json +import sqlite3 +import time +from dataclasses import dataclass +from functools import cached_property +from typing import Any + +from database.connection import DatabasePool + + +def _decode_jwt_payload(token: str) -> dict[str, Any]: + try: + _, claims, _ = token.split(".") + claims = claims + "=" * (4 - len(claims) % 4) if len(claims) % 4 else claims + return json.loads(base64.urlsafe_b64decode(claims)) # type: ignore[no-any-return] + except Exception: + return {} + + +@dataclass +class Session: + access_jwt: str + refresh_jwt: str + handle: str + did: str + pds: str + email: str | None = None + email_confirmed: bool = False + email_auth_factor: bool = False + active: bool = True + status: str | None = None + + @cached_property + def access_payload(self) -> dict[str, Any]: + return _decode_jwt_payload(self.access_jwt) + + @cached_property + def refresh_payload(self) -> dict[str, Any]: + return _decode_jwt_payload(self.refresh_jwt) + + def is_access_token_expired(self, buffer_seconds: int = 60) -> bool: + exp = self.access_payload.get("exp", 0) + return bool(time.time() >= (exp - buffer_seconds)) + + def is_refresh_token_expired(self, buffer_seconds: int = 60) -> bool: + exp = self.refresh_payload.get("exp", 0) + return bool(time.time() >= (exp - buffer_seconds)) + + @classmethod + def from_row(cls, row: sqlite3.Row) -> "Session": + return cls( + access_jwt=row["access_jwt"], + refresh_jwt=row["refresh_jwt"], + handle=row["handle"], + did=row["did"], + pds=row["pds"], + email=row["email"], + email_confirmed=bool(row["email_confirmed"]), + email_auth_factor=bool(row["email_auth_factor"]), + active=bool(row["active"]), + status=row["status"], + ) + + @classmethod + def from_dict(cls, data: dict[str, Any], pds: str) -> "Session": + return cls( + access_jwt=data["accessJwt"], + refresh_jwt=data["refreshJwt"], + handle=data["handle"], + did=data["did"], + pds=pds, + email=data.get("email"), + email_confirmed=data.get("emailConfirmed", False), + email_auth_factor=data.get("emailAuthFactor", False), + active=data.get("active", True), + status=data.get("status"), + ) + + +@dataclass +class IdentityInfo: + did: str + handle: str + pds: str + signing_key: str + + @classmethod + def from_row(cls, row: sqlite3.Row) -> "IdentityInfo": + return cls( + did=row["did"], + handle=row["handle"], + pds=row["pds"], + signing_key=row["signing_key"], + ) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "IdentityInfo": + return cls( + did=data["did"], + handle=data["handle"], + pds=data["pds"], + signing_key=data["signing_key"], + ) + + +class AtprotoStore: + def __init__( + self, + db: sqlite3.Connection, + identity_ttl: int = 12 * 60 * 60, + ) -> None: + self.db = db + self.db.row_factory = sqlite3.Row + self.identity_ttl = identity_ttl + + def get_session(self, did: str) -> Session | None: + row = self.db.execute( + "SELECT * FROM atproto_sessions WHERE did = ?", (did,) + ).fetchone() + return Session.from_row(row) if row else None + + def set_session(self, session: Session) -> None: + now = time.time() + self.db.execute( + """ + INSERT OR REPLACE INTO atproto_sessions + (did, pds, handle, access_jwt, refresh_jwt, email, email_confirmed, + email_auth_factor, active, status, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + session.did, + session.pds, + session.handle, + session.access_jwt, + session.refresh_jwt, + session.email, + session.email_confirmed, + session.email_auth_factor, + session.active, + session.status, + now, + ), + ) + self.db.commit() + + def get_session_by_pds(self, pds: str, identifier: str) -> Session | None: + row = self.db.execute( + """ + SELECT * FROM atproto_sessions + WHERE pds = ? AND (did = ? OR handle = ?) + """, + (pds, identifier, identifier), + ).fetchone() + return Session.from_row(row) if row else None + + def list_sessions_by_pds(self, pds: str) -> list[Session]: + rows = self.db.execute( + "SELECT * FROM atproto_sessions WHERE pds = ?", (pds,) + ).fetchall() + return [Session.from_row(row) for row in rows] + + def remove_session(self, did: str) -> None: + self.db.execute("DELETE FROM atproto_sessions WHERE did = ?", (did,)) + self.db.commit() + + def get_identity(self, identifier: str) -> IdentityInfo | None: + row = self.db.execute( + "SELECT * FROM atproto_identities WHERE identifier = ? AND created_at + ? > ?", + (identifier, self.identity_ttl, time.time()), + ).fetchone() + return IdentityInfo.from_row(row) if row else None + + def set_identity(self, identifier: str, identity: IdentityInfo) -> None: + now = time.time() + for key in (identifier, identity.did, identity.handle): + self.db.execute( + """ + INSERT OR REPLACE INTO atproto_identities + (identifier, did, handle, pds, signing_key, created_at) + VALUES (?, ?, ?, ?, ?, ?) + """, + ( + key, + identity.did, + identity.handle, + identity.pds, + identity.signing_key, + now, + ), + ) + self.db.commit() + + def remove_identity(self, identifier: str) -> None: + self.db.execute( + "DELETE FROM atproto_identities WHERE identifier = ?", (identifier,) + ) + self.db.commit() + + def cleanup_expired(self) -> None: + cutoff = time.time() - self.identity_ttl + self.db.execute( + "DELETE FROM atproto_identities WHERE created_at + ? < ?", + (self.identity_ttl, cutoff), + ) + self.db.commit() + + def flush_all(self) -> tuple[int, int]: + sessions = self.db.execute("SELECT COUNT(*) FROM atproto_sessions").fetchone()[ + 0 + ] + identities = self.db.execute( + "SELECT COUNT(*) FROM atproto_identities" + ).fetchone()[0] + self.db.execute("DELETE FROM atproto_sessions") + self.db.execute("DELETE FROM atproto_identities") + self.db.commit() + return sessions, identities + + +_store: AtprotoStore | None = None + + +def get_store(db: DatabasePool) -> AtprotoStore: + global _store + if _store is None: + _store = AtprotoStore(db.get_conn()) + return _store + + +def flush_caches() -> tuple[int, int]: + if _store is not None: + return _store.flush_all() + return 0, 0 diff --git a/atproto/xrpc.py b/atproto/xrpc.py new file mode 100644 --- /dev/null +++ b/atproto/xrpc.py @@ -0,0 +1,254 @@ +from dataclasses import dataclass +from typing import Any, TypeVar + +import httpx + +from atproto.store import AtprotoStore, IdentityInfo, Session +from util.util import LOGGER, normalize_service_url + + +class XRPCError(Exception): + def __init__( + self, + message: str, + status_code: int | None = None, + response_data: dict[str, Any] | None = None, + ) -> None: + super().__init__(message) + self.status_code = status_code + self.response_data = response_data + + +T = TypeVar("T") + + +@dataclass +class XRPCResponse: + data: dict[str, Any] + status_code: int + headers: dict[str, str] + + +class XRPCClient: + def __init__( + self, + pds_url: str, + store: AtprotoStore, + http: httpx.Client | None = None, + identifier: str | None = None, + password: str | None = None, + ) -> None: + self.pds_url: str = normalize_service_url(pds_url) + self.store: AtprotoStore = store + self.http: httpx.Client = http if http else httpx.Client() + + if identifier and password: + self.login(identifier, password) + + def login(self, identifier: str, password: str) -> Session: + cached = self.store.get_session_by_pds(self.pds_url, identifier) + if cached and not cached.is_refresh_token_expired(): + return cached + return self.create_session(identifier, password) + + def get_session(self, did: str) -> Session | None: + session = self.store.get_session(did) + if not session: + return None + if session.is_access_token_expired(): + if not session.is_refresh_token_expired(): + LOGGER.info("refreshing session for %s", session.did) + return self.refresh_session(session) + LOGGER.info("both tokens expired for %s, removing session", session.did) + self.store.remove_session(did) + raise ValueError( + "Both access and refresh tokens expired. Please login again." + ) + return session + + def create_session( + self, + identifier: str, + password: str, + auth_factor_token: str | None = None, + ) -> Session: + url = f"{self.pds_url}/xrpc/com.atproto.server.createSession" + payload: dict[str, str] = {"identifier": identifier, "password": password} + if auth_factor_token: + payload["authFactorToken"] = auth_factor_token + + response = self.http.post(url, json=payload, timeout=30) + + match response.status_code: + case 200: + pass + case 401: + raise ValueError("Invalid identifier or password") + case 400: + raise ValueError(f"Authentication failed: {response.json()}") + case _: + raise ValueError( + f"Authentication failed with status {response.status_code}" + ) + + session = Session.from_dict(response.json(), self.pds_url) + self.store.set_session(session) + LOGGER.info("Created session for %s (%s)", session.handle, session.did) + return session + + def refresh_session(self, session: Session) -> Session: + url = f"{self.pds_url}/xrpc/com.atproto.server.refreshSession" + headers = {"Authorization": f"Bearer {session.refresh_jwt}"} + + response = self.http.post(url, headers=headers, timeout=30) + + match response.status_code: + case 200: + pass + case 401: + error_data = response.json() if response.content else {} + raise ValueError(f"Refresh failed: {error_data}") + case 400: + raise ValueError(f"Refresh failed: {response.json()}") + case _: + raise ValueError(f"Refresh failed with status {response.status_code}") + + new_session = Session.from_dict(response.json(), self.pds_url) + self.store.set_session(new_session) + LOGGER.info( + "Refreshed session for %s (%s)", new_session.handle, new_session.did + ) + return new_session + + def get_access_token(self, did: str) -> str | None: + session = self.get_session(did) + return session.access_jwt if session else None + + def call( + self, + method: str, + params: dict[str, Any] | None = None, + data: dict[str, Any] | None = None, + did: str | None = None, + ) -> XRPCResponse: + url = f"{self.pds_url}/xrpc/{method}" + headers = ( + { + "Authorization": f"Bearer {self.get_access_token(did)}", + "Content-Type": "application/json", + } + if did + else {} + ) + + if params and data: + raise ValueError("Cannot specify both params and data") + + try: + if params: + response = self.http.get( + url, params=params, headers=headers, timeout=30 + ) + elif data: + response = self.http.post(url, json=data, headers=headers, timeout=30) + else: + response = self.http.get(url, headers=headers, timeout=30) + except httpx.RequestError as e: + raise XRPCError(f"Request failed: {e}") from e + + try: + response_data = response.json() if response.content else {} + except ValueError as e: + raise XRPCError( + f"Invalid JSON response: {e}", status_code=response.status_code + ) from e + + if response.status_code >= 400: + error_msg = response_data.get( + "message", f"Request failed with status {response.status_code}" + ) + raise XRPCError( + error_msg, status_code=response.status_code, response_data=response_data + ) + + return XRPCResponse( + data=response_data, + status_code=response.status_code, + headers=dict(response.headers), + ) + + def upload_blob( + self, + blob: bytes, + content_type: str, + did: str, + ) -> dict[str, Any]: + token = self.get_access_token(did) + if not token: + raise ValueError(f"No valid session found for {did}") + + url = f"{self.pds_url}/xrpc/com.atproto.repo.uploadBlob" + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": content_type, + } + + try: + response = self.http.post(url, content=blob, headers=headers, timeout=60) + except httpx.RequestError as e: + raise XRPCError(f"Blob upload request failed: {e}") from e + + if response.status_code != 200: + error_data = response.json() if response.content else {} + raise XRPCError( + f"Blob upload failed: {response.status_code}", + status_code=response.status_code, + response_data=error_data, + ) + + try: + result: dict[str, Any] = response.json() + except ValueError as e: + raise XRPCError(f"Invalid JSON response from blob upload: {e}") from e + + return result + + +def resolve_identity( + identifier: str, store: AtprotoStore, http: httpx.Client | None = None +) -> IdentityInfo: + import env + + cached = store.get_identity(identifier) + if cached: + return cached + + url = f"{env.SLINGSHOT_URL}/xrpc/com.bad-example.identity.resolveMiniDoc" + client = http if http else httpx.Client() + + try: + response = client.get(url, params={"identifier": identifier}, timeout=10) + response.raise_for_status() + + try: + data = response.json() + except ValueError as e: + raise ValueError( + f"Invalid JSON response from identity resolver: {e}" + ) from e + + match response.status_code: + case 200: + identity = IdentityInfo.from_dict(data) + store.set_identity(identifier, identity) + return identity + case 404: + raise ValueError(f"Identity not found: {identifier}") + case _: + error_msg = data.get( + "message", + f"Identity resolver returned status {response.status_code}", + ) + raise ValueError(error_msg) + except httpx.RequestError as e: + raise ValueError(f"Failed to resolve identity {identifier}: {e}") from e diff --git a/bluesky/__init__.py b/bluesky/__init__.py new file mode 100644 --- /dev/null +++ b/bluesky/__init__.py diff --git a/bluesky/client.py b/bluesky/client.py new file mode 100644 --- /dev/null +++ b/bluesky/client.py @@ -0,0 +1,359 @@ +import logging +from datetime import UTC, datetime +from typing import Any, cast + +import httpx + +from atproto.models import ( + AtUri, + CreateRecordResponse, + Facet, + ImageEmbed, + PostGate, + PostRecord, + RecordEmbed, + RecordWithMediaEmbed, + ReplyRef, + RepostRecord, + SelfLabels, + StrongRef, + ThreadGate, + VideoEmbed, +) +from atproto.store import AtprotoStore +from atproto.xrpc import XRPCClient, XRPCError, resolve_identity +from util.util import normalize_service_url + + +logger = logging.getLogger(__name__) + + +class BlueskyClient: + def __init__( + self, + pds_url: str, + store: AtprotoStore, + identifier: str, + http: httpx.Client | None = None, + password: str | None = None, + ) -> None: + self.pds_url: str = normalize_service_url(pds_url) + self.store: AtprotoStore = store + + identity = resolve_identity(identifier, store, http) + self.did: str = identity.did + self.xrpc: XRPCClient = XRPCClient(pds_url, store, http, self.did, password) + + def _get_timestamp(self, time_iso: str | None = None) -> str: + if time_iso: + return time_iso + return datetime.now(UTC).isoformat().replace("+00:00", "Z") + + def _upload_blob(self, data: bytes, content_type: str) -> dict[str, Any]: + return self.xrpc.upload_blob(data, content_type, self.did) + + def send_post( + self, + text: str, + facets: list[Facet] | None = None, + embed: dict[str, Any] | None = None, + reply_to: ReplyRef | None = None, + labels: SelfLabels | None = None, + langs: list[str] | None = None, + time_iso: str | None = None, + ) -> CreateRecordResponse: + record = PostRecord( + text=text, + facets=facets, + embed=embed, + reply=reply_to, + labels=labels, + langs=langs, + created_at=self._get_timestamp(time_iso), + ) + + response = self.xrpc.call( + "com.atproto.repo.createRecord", + data={ + "repo": self.did, + "collection": "app.bsky.feed.post", + "record": record.to_dict(), + }, + did=self.did, + ) + + return CreateRecordResponse.from_dict(response.data) + + def send_images( + self, + text: str, + images: list[ImageEmbed], + facets: list[Facet] | None = None, + embed: dict[str, Any] | None = None, + reply_to: ReplyRef | None = None, + labels: SelfLabels | None = None, + langs: list[str] | None = None, + time_iso: str | None = None, + ) -> CreateRecordResponse: + image_refs: list[dict[str, Any]] = [] + for img in images[:4]: + blob_ref = self._upload_blob(img.image, "image/jpeg") + image_data = img.to_dict(blob_ref["blob"]) + image_refs.append(image_data) + + image_embed: dict[str, Any] = { + "$type": "app.bsky.embed.images", + "images": image_refs, + } + + if embed: + combined_embed: dict[str, Any] = { + "$type": "app.bsky.embed.recordWithMedia", + "record": embed, + "media": image_embed, + } + return self.send_post( + text=text, + facets=facets, + embed=combined_embed, + reply_to=reply_to, + labels=labels, + langs=langs, + time_iso=time_iso, + ) + + return self.send_post( + text=text, + facets=facets, + embed=image_embed, + reply_to=reply_to, + labels=labels, + langs=langs, + time_iso=time_iso, + ) + + def send_video( + self, + text: str, + video: bytes, + alt: str | None = None, + aspect_ratio: tuple[int, int] | None = None, + facets: list[Facet] | None = None, + embed: dict[str, Any] | None = None, + reply_to: ReplyRef | None = None, + labels: SelfLabels | None = None, + langs: list[str] | None = None, + time_iso: str | None = None, + ) -> CreateRecordResponse: + blob_ref = self._upload_blob(video, "video/mp4") + + video_embed = VideoEmbed( + video=video, + alt=alt, + aspect_ratio=aspect_ratio, + ) + video_embed_dict = video_embed.to_dict(blob_ref["blob"]) + + if embed: + combined_embed: dict[str, Any] = { + "$type": "app.bsky.embed.recordWithMedia", + "record": embed, + "media": video_embed_dict, + } + return self.send_post( + text=text, + facets=facets, + embed=combined_embed, + reply_to=reply_to, + labels=labels, + langs=langs, + time_iso=time_iso, + ) + + return self.send_post( + text=text, + facets=facets, + embed=video_embed_dict, + reply_to=reply_to, + labels=labels, + langs=langs, + time_iso=time_iso, + ) + + def send_quote( + self, + text: str, + quoted_uri: str, + quoted_cid: str, + facets: list[Facet] | None = None, + embed_media: ImageEmbed | VideoEmbed | None = None, + embed_blob_ref: dict[str, Any] | None = None, + reply_to: ReplyRef | None = None, + labels: SelfLabels | None = None, + langs: list[str] | None = None, + time_iso: str | None = None, + ) -> CreateRecordResponse: + quoted_ref = StrongRef(uri=quoted_uri, cid=quoted_cid) + + if embed_media and embed_blob_ref: + embed: dict[str, Any] = RecordWithMediaEmbed( + record=quoted_ref, + media=embed_media, + media_blob_ref=embed_blob_ref, + ).to_dict() + else: + embed = RecordEmbed(record=quoted_ref).to_dict() + + return self.send_post( + text=text, + facets=facets, + embed=embed, + reply_to=reply_to, + labels=labels, + langs=langs, + time_iso=time_iso, + ) + + def repost( + self, subject_uri: str, subject_cid: str, time_iso: str | None = None + ) -> CreateRecordResponse: + subject = StrongRef(uri=subject_uri, cid=subject_cid) + record = RepostRecord( + subject=subject, + created_at=self._get_timestamp(time_iso), + ) + + record_dict = record.to_dict() + + response = self.xrpc.call( + "com.atproto.repo.createRecord", + data={ + "repo": self.did, + "collection": "app.bsky.feed.repost", + "record": record_dict, + }, + did=self.did, + ) + + return CreateRecordResponse.from_dict(response.data) + + def delete_post(self, post_uri: str) -> None: + _, _, rkey = AtUri.record_uri(post_uri) + + self.xrpc.call( + "com.atproto.repo.deleteRecord", + data={ + "repo": self.did, + "collection": "app.bsky.feed.post", + "rkey": rkey, + }, + did=self.did, + ) + + def delete_repost(self, repost_uri: str) -> None: + _, _, rkey = AtUri.record_uri(repost_uri) + + self.xrpc.call( + "com.atproto.repo.deleteRecord", + data={ + "repo": self.did, + "collection": "app.bsky.feed.repost", + "rkey": rkey, + }, + did=self.did, + ) + + def create_threadgate( + self, post_uri: str, allow_gates: list[str] | None + ) -> CreateRecordResponse: + allow: list[dict[str, Any]] = [] + if allow_gates: + for gate in allow_gates: + match gate: + case "mentioned": + allow.append({"$type": "app.bsky.feed.threadgate#mentionRule"}) + case "following": + allow.append( + {"$type": "app.bsky.feed.threadgate#followingRule"} + ) + case "followers": + allow.append({"$type": "app.bsky.feed.threadgate#followerRule"}) + + threadgate = ThreadGate( + allow=allow, post=post_uri, created_at=self._get_timestamp() + ) + + _, _, rkey = AtUri.record_uri(post_uri) + + response = self.xrpc.call( + "com.atproto.repo.createRecord", + data={ + "repo": self.did, + "collection": "app.bsky.feed.threadgate", + "record": threadgate.to_dict(), + "rkey": rkey, + }, + did=self.did, + ) + + return CreateRecordResponse.from_dict(response.data) + + def create_postgate( + self, post_uri: str, quote_gate: bool = True + ) -> CreateRecordResponse: + postgate = PostGate( + post=post_uri, + created_at=self._get_timestamp(), + embedding_rules=[{"$type": "app.bsky.feed.postgate#disableRule"}] + if quote_gate + else None, + ) + + _, _, rkey = AtUri.record_uri(post_uri) + + response = self.xrpc.call( + "com.atproto.repo.createRecord", + data={ + "repo": self.did, + "collection": "app.bsky.feed.postgate", + "record": postgate.to_dict(), + "rkey": rkey, + }, + did=self.did, + ) + + return CreateRecordResponse.from_dict(response.data) + + def create_gates( + self, + post_uri: str, + thread_gate: list[str] | None, + quote_gate: bool, + ) -> tuple[CreateRecordResponse | None, CreateRecordResponse | None]: + threadgate_response: CreateRecordResponse | None = None + postgate_response: CreateRecordResponse | None = None + + if thread_gate is not None: + threadgate_response = self.create_threadgate(post_uri, thread_gate) + + if quote_gate: + postgate_response = self.create_postgate(post_uri, quote_gate) + + return threadgate_response, postgate_response + + def get_post(self, uri: str) -> dict[str, Any] | None: + try: + response = self.xrpc.call( + "app.bsky.feed.getPosts", + params={"uris": [uri]}, + ) + posts = cast(list[dict[str, Any]], response.data.get("posts", [])) + return posts[0] if posts else None + except XRPCError as e: + if e.status_code == 404: + return None + logger.warning("Failed to get post %s: %s", uri, e) + return None + except Exception as e: + logger.warning("Unexpected error getting post %s: %s", uri, e) + return None diff --git a/bluesky/info.py b/bluesky/info.py new file mode 100644 --- /dev/null +++ b/bluesky/info.py @@ -0,0 +1,54 @@ +from abc import ABC, abstractmethod +from typing import Any + +from atproto.store import AtprotoStore +from atproto.xrpc import resolve_identity +from cross.service import Service +from util.util import normalize_service_url + + +SERVICE = "https://bsky.app" + + +def validate_and_transform(data: dict[str, Any]) -> None: + if not data.get("handle") and not data.get("did"): + raise KeyError("no 'handle' or 'did' specified for bluesky!") + + if "did" in data: + did = str(data["did"]) # only did:web and did:plc are supported + if not did.startswith("did:plc:") and not did.startswith("did:web:"): + raise ValueError( + f"Invalid DID format: {did}! Only did:plc: and did:web: are supported." + ) + + if "pds" in data: + data["pds"] = normalize_service_url(data["pds"]) + + +class BlueskyService(ABC, Service): + pds: str + did: str + _store: AtprotoStore + + def _init_identity(self) -> None: + handle, did, pds = self.get_identity_options() + if did: + self.did = did + if pds: + self.pds = pds + + if not did: + if not handle: + raise KeyError("No did: or atproto handle provided!") + self.log.info("Resolving ATP identity for %s...", handle) + identity = resolve_identity(handle, self._store) + self.did = identity.did + + if not pds: + self.log.info("Resolving PDS for %s...", self.did) + identity = resolve_identity(self.did, self._store) + self.pds = identity.pds + + @abstractmethod + def get_identity_options(self) -> tuple[str | None, str | None, str | None]: + pass diff --git a/bluesky/input.py b/bluesky/input.py new file mode 100644 --- /dev/null +++ b/bluesky/input.py @@ -0,0 +1,323 @@ +import asyncio +import json +import re +from abc import ABC +from dataclasses import dataclass, field +from typing import Any, cast, override + +import httpx +import websockets + +import env +from atproto.models import AtUri +from atproto.store import get_store +from bluesky.info import SERVICE, BlueskyService, validate_and_transform +from bluesky.richtext import richtext_to_tokens +from cross.attachments import ( + LabelsAttachment, + LanguagesAttachment, + MediaAttachment, + QuoteAttachment, + RemoteUrlAttachment, +) +from cross.media import Blob, download_blob +from cross.post import Post, PostRef +from cross.service import InputService +from database.connection import DatabasePool + + +@dataclass(kw_only=True) +class BlueskyInputOptions: + handle: str | None = None + did: str | None = None + pds: str | None = None + filters: list[re.Pattern[str]] = field(default_factory=lambda: []) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "BlueskyInputOptions": + validate_and_transform(data) + + if "filters" in data: + data["filters"] = [re.compile(r) for r in data["filters"]] + + return BlueskyInputOptions(**data) + + +class BlueskyBaseInputService(BlueskyService, InputService, ABC): + def __init__(self, db: DatabasePool, http: httpx.Client) -> None: + super().__init__(SERVICE, db) + self.http = http + + def _on_post(self, record: dict[str, Any]): + post_uri = cast(str, record["$xpost.strongRef"]["uri"]) + post_cid = cast(str, record["$xpost.strongRef"]["cid"]) + + self.log.info("Processing new post: %s", post_uri) + + if self._is_post_crossposted(self.url, self.did, post_uri): + self.log.info( + "Skipping %s, already crossposted", + post_uri, + ) + return + + parent_uri = cast( + str, None if not record.get("reply") else record["reply"]["parent"]["uri"] + ) + parent = None + if parent_uri: + parent = self._get_post(self.url, self.did, parent_uri) + if not parent: + self.log.info( + "Skipping %s, parent %s not found in db", post_uri, parent_uri + ) + return + + tokens = richtext_to_tokens(record["text"], record.get("facets", [])) + post = Post( + id=post_uri, + author=self.did, + service=self.url, + parent_id=parent_uri, + tokens=tokens, + ) + + did, _, rid = AtUri.record_uri(post_uri) + post.attachments.put( + RemoteUrlAttachment(url=f"https://bsky.app/profile/{did}/post/{rid}") + ) + + embed: dict[str, Any] = record.get("embed", {}) + blob_urls: list[tuple[str, str, str | None]] = [] + + def handle_embeds(embed: dict[str, Any]) -> str | None: + nonlocal blob_urls, post + match cast(str, embed["$type"]): + case "app.bsky.embed.record" | "app.bsky.embed.recordWithMedia": + rcrd = ( + embed["record"]["record"] + if embed["record"].get("record") + else embed["record"] + ) + did, collection, _ = AtUri.record_uri(rcrd["uri"]) + if collection != "app.bsky.feed.post": + return f"Unhandled record collection {collection}" + if did != self.did: + return "" + + rquote = self._get_post(self.url, did, rcrd["uri"]) + if not rquote: + return f"Quote {rcrd['uri']} not found in the db" + post.attachments.put( + QuoteAttachment(quoted_id=rcrd["uri"], quoted_user=did) + ) + + if embed.get("media"): + return handle_embeds(embed["media"]) + case "app.bsky.embed.images": + for image in embed["images"]: + blob_cid = image["image"]["ref"]["$link"] + url = f"{self.pds}/xrpc/com.atproto.sync.getBlob?did={self.did}&cid={blob_cid}" + blob_urls.append((url, blob_cid, image.get("alt"))) + case "app.bsky.embed.video": + blob_cid = embed["video"]["ref"]["$link"] + url = f"{self.pds}/xrpc/com.atproto.sync.getBlob?did={self.did}&cid={blob_cid}" + blob_urls.append((url, blob_cid, embed.get("alt"))) + case _: + self.log.warning(f"Unhandled embed type {embed['$type']}") + return None + + if embed: + fexit = handle_embeds(embed) + if fexit is not None: + self.log.info("Skipping %s! %s", post_uri, fexit) + return + + if blob_urls: + blobs: list[Blob] = [] + for url, cid, alt in blob_urls: + self.log.info("Downloading %s...", cid) + blob: Blob | None = download_blob(url, alt, client=self.http) + if not blob: + self.log.error( + "Skipping %s! Failed to download blob %s.", post_uri, cid + ) + return + blobs.append(blob) + post.attachments.put(MediaAttachment(blobs=blobs)) + + if "langs" in record: + post.attachments.put(LanguagesAttachment(langs=record["langs"])) + if "labels" in record: + post.attachments.put( + LabelsAttachment( + labels=[ + label["val"].replace("-", " ") for label in record["values"] + ] + ), + ) + + if parent: + self._insert_post( + { + "user": self.did, + "service": self.url, + "identifier": post_uri, + "parent": parent["id"], + "root": parent["id"] if not parent["root"] else parent["root"], + "extra_data": json.dumps({"cid": post_cid}), + } + ) + else: + self._insert_post( + { + "user": self.did, + "service": self.url, + "identifier": post_uri, + "extra_data": json.dumps({"cid": post_cid}), + } + ) + + self.log.info("Post stored in DB: %s", post_uri) + + for out in self.outputs: + self.submitter(lambda: out.accept_post(post)) + + def _on_repost(self, record: dict[str, Any]): + post_uri = cast(str, record["$xpost.strongRef"]["uri"]) + post_cid = cast(str, record["$xpost.strongRef"]["cid"]) + + self.log.info("Processing repost: %s", post_uri) + + reposted_uri = cast(str, record["subject"]["uri"]) + reposted = self._get_post(self.url, self.did, reposted_uri) + if not reposted: + self.log.info( + "Skipping repost '%s' as reposted post '%s' was not found in the db.", + post_uri, + reposted_uri, + ) + return + + self._insert_post( + { + "user": self.did, + "service": self.url, + "identifier": post_uri, + "reposted": reposted["id"], + "extra_data": json.dumps({"cid": post_cid}), + } + ) + + self.log.info("Repost stored in DB: %s", post_uri) + + repost_ref = PostRef(id=post_uri, author=self.did, service=self.url) + reposted_ref = PostRef(id=reposted_uri, author=self.did, service=self.url) + for out in self.outputs: + self.submitter(lambda: out.accept_repost(repost_ref, reposted_ref)) + + def _on_delete_post(self, post_id: str, repost: bool): + self.log.info("Processing delete for %s (repost: %s)...", post_id, repost) + post = self._get_post(self.url, self.did, post_id) + if not post: + self.log.warning("Post not found in DB: %s", post_id) + return + + post_ref = PostRef(id=post_id, author=self.did, service=self.url) + if repost: + self.log.info("Deleting repost: %s", post_id) + for output in self.outputs: + self.submitter(lambda: output.delete_repost(post_ref)) + else: + self.log.info("Deleting post: %s", post_id) + for output in self.outputs: + self.submitter(lambda: output.delete_post(post_ref)) + self.submitter(lambda: self._delete_post_by_id(post["id"])) + self.log.info("Delete successful for %s", post_id) + + +class BlueskyJetstreamInputService(BlueskyBaseInputService): + def __init__( + self, + db: DatabasePool, + http: httpx.Client, + options: BlueskyInputOptions, + ) -> None: + super().__init__(db, http) + self.options: BlueskyInputOptions = options + self._store = get_store(db) + self._init_identity() + + @override + def get_identity_options(self) -> tuple[str | None, str | None, str | None]: + return (self.options.handle, self.options.did, self.options.pds) + + def _accept_msg(self, msg: websockets.Data) -> None: + data: dict[str, Any] = cast(dict[str, Any], json.loads(msg)) + if data.get("did") != self.did: + return + commit: dict[str, Any] | None = data.get("commit") + if not commit: + return + + commit_type: str = cast(str, commit["operation"]) + match commit_type: + case "create": + record: dict[str, Any] = cast(dict[str, Any], commit["record"]) + record["$xpost.strongRef"] = { + "cid": commit["cid"], + "uri": f"at://{self.did}/{commit['collection']}/{commit['rkey']}", + } + + match cast(str, commit["collection"]): + case "app.bsky.feed.post": + self._on_post(record) + case "app.bsky.feed.repost": + self._on_repost(record) + case _: + pass + case "delete": + post_id: str = ( + f"at://{self.did}/{commit['collection']}/{commit['rkey']}" + ) + match cast(str, commit["collection"]): + case "app.bsky.feed.post": + self._on_delete_post(post_id, False) + case "app.bsky.feed.repost": + self._on_delete_post(post_id, True) + case _: + pass + case _: + pass + + @override + async def listen(self): + url = env.JETSTREAM_URL + "?" + url += "wantedCollections=app.bsky.feed.post" + url += "&wantedCollections=app.bsky.feed.repost" + url += f"&wantedDids={self.did}" + + async for ws in websockets.connect( + url, + ping_interval=20, + ping_timeout=10, + close_timeout=5, + ): + try: + self.log.info("Listening to %s...", env.JETSTREAM_URL) + + async def listen_for_messages(): + async for msg in ws: + self.submitter(lambda: self._accept_msg(msg)) + + listen = asyncio.create_task(listen_for_messages()) + + _ = await asyncio.gather(listen) + except websockets.ConnectionClosedError as e: + self.log.error(e, stack_info=True, exc_info=True) + self.log.info("Reconnecting to %s...", env.JETSTREAM_URL) + continue + except TimeoutError as e: + self.log.error("Connection timeout: %s", e) + self.log.info("Reconnecting to %s...", env.JETSTREAM_URL) + continue diff --git a/bluesky/output.py b/bluesky/output.py new file mode 100644 --- /dev/null +++ b/bluesky/output.py @@ -0,0 +1,658 @@ +import json +import re +from dataclasses import dataclass +from typing import Any, override + +import httpx + +import misskey.mfm as mfm +from atproto.models import ( + Facet, + ImageEmbed, + RecordEmbed, + ReplyRef, + SelfLabel, + SelfLabels, + StrongRef, +) +from atproto.store import get_store +from bluesky.client import BlueskyClient +from bluesky.info import SERVICE, BlueskyService, validate_and_transform +from bluesky.richtext import tokens_to_richtext +from cross.attachments import ( + LabelsAttachment, + LanguagesAttachment, + MediaAttachment, + QuoteAttachment, + RemoteUrlAttachment, + SensitiveAttachment, +) +from cross.media import Blob, compress_image, convert_to_mp4, get_media_meta +from cross.post import Post, PostRef +from cross.service import OutputService +from cross.tokens import LinkToken, TextToken, Token +from database.connection import DatabasePool +from util.splitter import TokenSplitter + + +ALLOWED_GATES: list[str] = ["mentioned", "following", "followers"] + +ADULT_PATTERN = re.compile(r"\b(adult|sexual|nsfw)\b", re.IGNORECASE) +PORN_PATTERN = re.compile(r"\b(porn|explicit)\b", re.IGNORECASE) + + +@dataclass(kw_only=True) +class BlueskyOutputOptions: + handle: str | None = None + did: str | None = None + pds: str | None = None + password: str = "" + quote_gate: bool = False + thread_gate: list[str] | None = None + encode_videos: bool = True + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "BlueskyOutputOptions": + validate_and_transform(data) + + if "password" not in data: + raise KeyError("password is required for bluesky") + + if "quote_gate" in data: + data["quote_gate"] = bool(data["quote_gate"]) + + if ( + "thread_gate" in data + and isinstance(data["thread_gate"], list) + and any(v not in ALLOWED_GATES for v in data["thread_gate"]) + ): + raise ValueError( + f"'thread_gate' only accepts {', '.join(ALLOWED_GATES)}, " + f"got: {data['thread_gate']}" + ) + + if "encode_videos" in data: + data["encode_videos"] = bool(data["encode_videos"]) + + return BlueskyOutputOptions(**data) + + +class BlueskyOutputService(BlueskyService, OutputService): + def __init__( + self, db: DatabasePool, http: httpx.Client, options: BlueskyOutputOptions + ) -> None: + super().__init__(SERVICE, db) + self.http = http + self.options: BlueskyOutputOptions = options + + self._store = get_store(db) + self._init_identity() + + self._client = BlueskyClient( + self.pds, + self._store, + self.did, + http, + self.options.password, + ) + self.options.password = "" + self.log.info("Logged in as %s", self.did) + + @override + def get_identity_options(self) -> tuple[str | None, str | None, str | None]: + return (self.options.handle, self.options.did, self.options.pds) + + def _split_attachments( + self, attachments: list[Blob] + ) -> tuple[list[Blob], list[Blob]]: + supported: list[Blob] = [] + unsupported: list[Blob] = [] + + for blob in attachments: + if blob.mime.startswith(("image/", "video/")): + supported.append(blob) + else: + unsupported.append(blob) + + return supported, unsupported + + def _split_media_per_post( + self, + token_blocks: list[list[Any]], + media: list[Blob], + ) -> list[tuple[list[Any], list[Blob]]]: + posts: list[dict[str, Any]] = [ + {"tokens": block, "attachments": []} for block in token_blocks + ] + available_indices: list[int] = list(range(len(posts))) + current_image_post_idx: int | None = None + + def make_blank_post() -> dict[str, Any]: + return {"tokens": [], "attachments": []} + + def pop_next_empty_index() -> int: + if available_indices: + return available_indices.pop(0) + new_idx = len(posts) + posts.append(make_blank_post()) + return new_idx + + for blob in media: + if blob.mime.startswith("video/"): + current_image_post_idx = None + idx = pop_next_empty_index() + posts[idx]["attachments"].append(blob) + elif blob.mime.startswith("image/"): + if ( + current_image_post_idx is not None + and len(posts[current_image_post_idx]["attachments"]) < 4 + ): + posts[current_image_post_idx]["attachments"].append(blob) + else: + idx = pop_next_empty_index() + posts[idx]["attachments"].append(blob) + current_image_post_idx = idx + + result: list[tuple[list[Any], list[Blob]]] = [] + for p in posts: + result.append((p["tokens"], p["attachments"])) + + return result + + def _build_labels( + self, + spoiler: str | None, + is_sensitive: bool, + ) -> SelfLabels | None: + unique_labels: set[str] = set() + + if spoiler: + unique_labels.add("graphic-media") + + if PORN_PATTERN.search(spoiler): + unique_labels.add("porn") + elif ADULT_PATTERN.search(spoiler): + unique_labels.add("sexual") + + if is_sensitive: + unique_labels.add("graphic-media") + + if not unique_labels: + return None + + return SelfLabels(values=[SelfLabel(val=label) for label in unique_labels]) + + @override + def accept_post(self, post: Post): + self.log.info( + "Accepting post %s (author: %s, service: %s)...", + post.id, + post.author, + post.service, + ) + reply_to: ReplyRef | None = None + new_root_id: int | None = None + new_parent_id: int | None = None + + if post.parent_id: + parent = self._get_post(post.service, post.author, post.parent_id) + if not parent: + self.log.error("Parent post not found in DB: %s", post.parent_id) + return + + thread = self._find_mapped_thread( + parent["identifier"], + parent["service"], + parent["user"], + self.url, + self.did, + ) + if not thread: + self.log.error( + "Failed to find thread tuple in the database for parent: %s", + post.parent_id, + ) + return + + root_uri, reply_uri, root_db_id, reply_db_id = thread + + root_post = self._get_post(self.url, self.did, root_uri) + reply_post = self._get_post(self.url, self.did, reply_uri) + + if not root_post or not reply_post: + self.log.error("Failed to fetch parent posts from database!") + return + + try: + root_cid_data = root_post["extra_data"] + root_cid = ( + json.loads(root_cid_data).get("cid", "") if root_cid_data else "" + ) + reply_cid_data = reply_post["extra_data"] + reply_cid = ( + json.loads(reply_cid_data).get("cid", "") if reply_cid_data else "" + ) + except (json.JSONDecodeError, AttributeError, KeyError): + self.log.error("Failed to parse CID from database!") + return + + root_ref = StrongRef(uri=root_uri, cid=root_cid) + reply_ref = StrongRef(uri=reply_uri, cid=reply_cid) + reply_to = ReplyRef(root=root_ref, parent=reply_ref) + new_root_id = root_db_id + new_parent_id = reply_db_id + + labels_attachment = post.attachments.get(LabelsAttachment) + spoiler: str | None = ( + labels_attachment.labels[0] + if labels_attachment and labels_attachment.labels + else None + ) + sensitive_attachment = post.attachments.get(SensitiveAttachment) + is_sensitive = sensitive_attachment.sensitive if sensitive_attachment else False + + labels = self._build_labels(spoiler, is_sensitive) + + langs: list[str] | None = None + langs_attachment = post.attachments.get(LanguagesAttachment) + if langs_attachment and langs_attachment.langs: + langs = langs_attachment.langs[:3] + + media_attachment = post.attachments.get(MediaAttachment) + all_media = media_attachment.blobs if media_attachment else [] + supported_media, unsupported_media = self._split_attachments(all_media) + + tokens: list[Token] = [] + + if spoiler: + tokens.append(TextToken(text=f"[{spoiler}]\n\n")) + + tokens.extend(post.tokens) + + if unsupported_media: + tokens.append(TextToken(text="\n")) + for attachment in unsupported_media: + url = ( + attachment.url + if hasattr(attachment, "url") and attachment.url + else attachment.name or "unknown" + ) + + name = "💾 file" + if attachment.name: + if attachment.mime.startswith("audio/"): + name = "🎵 " + attachment.name + elif attachment.mime.startswith("text/"): + name = "📄 " + attachment.name + else: + name = "💾 " + attachment.name + + if len(name) > 28: + name = name[: 28 - 1] + "…" + + tokens.append(LinkToken(href=url, label=f"[{name}]")) + tokens.append(TextToken(text=" ")) + + if post.text_type == "text/x.misskeymarkdown": + tokens, status = mfm.strip_mfm(tokens) + remote_url = post.attachments.get(RemoteUrlAttachment) + if status and remote_url and remote_url.url: + tokens.append(TextToken(text="\n")) + tokens.append( + LinkToken( + href=remote_url.url, label="[Post contains MFM, see original]" + ) + ) + + quote_attachment = post.attachments.get(QuoteAttachment) + quoted_cid: str | None = None + quoted_uri: str | None = None + if quote_attachment: + if quote_attachment.quoted_user != post.author: + self.log.info("Quoted other user, skipping quote!") + return + + quoted_post = self._get_post( + post.service, post.author, quote_attachment.quoted_id + ) + if not quoted_post: + self.log.error("Failed to find quoted post in the database!") + else: + quoted_mappings = self._get_mappings( + quoted_post["id"], self.url, self.did + ) + if not quoted_mappings: + self.log.error("Failed to find mappings for quoted post!") + else: + bluesky_quoted_post = self._get_post( + self.url, self.did, quoted_mappings[0]["identifier"] + ) + if not bluesky_quoted_post: + self.log.error("Failed to find Bluesky quoted post!") + else: + quoted_cid_data = bluesky_quoted_post["extra_data"] + quoted_cid = ( + json.loads(quoted_cid_data).get("cid", "") + if quoted_cid_data + else "" + ) + quoted_uri = quoted_mappings[0]["identifier"] + + splitter = TokenSplitter(max_chars=300, max_link_len=30) + token_blocks = splitter.split(tokens) + + if token_blocks is None: + self.log.error( + "Skipping '%s' as it contains links/tags that are too long!", post.id + ) + return + + for blob in supported_media: + if blob.mime.startswith("image/") and len(blob.io) > 2_000_000: + self.log.error( + "Skipping post '%s', image too large!", + post.id, + ) + return + if blob.mime.startswith("video/"): + if blob.mime != "video/mp4" and not self.options.encode_videos: + self.log.info( + "Video is not mp4, but encoding is disabled. Skipping '%s'...", + post.id, + ) + return + if len(blob.io) > 100_000_000: + self.log.error( + "Skipping post '%s', video too large!", + post.id, + ) + return + + baked_media = self._split_media_per_post( + [list(block) for block in token_blocks], + supported_media, + ) + + precomputed_richtexts: list[tuple[str, list[Facet]]] = [] + for block in token_blocks: + result = tokens_to_richtext(block) + if result is None: + self.log.error( + "Skipping '%s' as it contains invalid rich text types!", + post.id, + ) + return + precomputed_richtexts.append(result) + + created_records: list[tuple[str, str]] = [] + post_root_ref: StrongRef | None = None + previous_reply_ref: StrongRef | None = None + + richtext_index = 0 + + for i, (block_tokens, attachments) in enumerate(baked_media): + if block_tokens and richtext_index < len(precomputed_richtexts): + text, facets = precomputed_richtexts[richtext_index] + richtext_index += 1 + else: + text = "" + facets = [] + + current_reply_to: ReplyRef | None = None + if i == 0: + current_reply_to = reply_to + elif previous_reply_ref and post_root_ref: + current_reply_to = ReplyRef( + root=post_root_ref, parent=previous_reply_ref + ) + + embed: dict[str, Any] | None = None + if i == 0 and quoted_uri and quoted_cid: + if attachments and attachments[0].mime.startswith("image/"): + embed = RecordEmbed( + record=StrongRef(uri=quoted_uri, cid=quoted_cid) + ).to_dict() + else: + embed = RecordEmbed( + record=StrongRef(uri=quoted_uri, cid=quoted_cid) + ).to_dict() + + if not attachments: + response = self._client.send_post( + text=text or " ", + facets=facets or None, + embed=embed, + reply_to=current_reply_to, + labels=labels, + langs=langs, + ) + elif attachments[0].mime.startswith("image/"): + images: list[ImageEmbed] = [] + for img_blob in attachments[:4]: + image_io = img_blob.io + if len(image_io) > 1_000_000: + self.log.info("Compressing %s...", img_blob.name or "image") + compressed = compress_image(img_blob) + image_io = compressed.io + + try: + meta = get_media_meta(image_io) + aspect_ratio = (meta.width, meta.height) + except Exception as e: + self.log.error(e) + aspect_ratio = None + + images.append( + ImageEmbed( + image=image_io, + alt=img_blob.alt, + aspect_ratio=aspect_ratio, + ) + ) + + response = self._client.send_images( + text=text or "", + images=images, + facets=facets or None, + embed=embed, + reply_to=current_reply_to, + labels=labels, + langs=langs, + ) + else: + video_blob = attachments[0] + video_io = video_blob.io + + if video_blob.mime != "video/mp4": + self.log.info("Converting %s to mp4...", video_blob.name or "video") + converted = convert_to_mp4(video_blob) + video_io = converted.io + + try: + meta = get_media_meta(video_io) + aspect_ratio = (meta.width, meta.height) + duration = meta.duration + except Exception as e: + self.log.error(e) + aspect_ratio = None + duration = None + + if duration and duration > 180: + self.log.info( + "Skipping post '%s', video too long (%.1f > 180s)!", + post.id, + duration, + ) + return + + response = self._client.send_video( + text=text or "", + video=video_io, + alt=video_blob.alt, + aspect_ratio=aspect_ratio, + embed=embed, + reply_to=current_reply_to, + labels=labels, + langs=langs, + ) + + created_records.append((response.uri, response.cid)) + + if post_root_ref is None: + post_root_ref = StrongRef(uri=response.uri, cid=response.cid) + previous_reply_ref = StrongRef(uri=response.uri, cid=response.cid) + + if i == 0: + self._client.create_gates( + response.uri, + self.options.thread_gate, + self.options.quote_gate, + ) + + db_post = self._get_post(post.service, post.author, post.id) + if not db_post: + self.log.error("Post not found in database!") + return + + if new_root_id is None or new_parent_id is None: + self._insert_post( + { + "user": self.did, + "service": self.url, + "identifier": created_records[0][0], + "parent": None, + "root": None, + "reposted": None, + "extra_data": json.dumps({"cid": created_records[0][1]}), + "crossposted": 1, + } + ) + new_post = self._get_post(self.url, self.did, created_records[0][0]) + if not new_post: + raise ValueError("Inserted post not found!") + new_root_id = new_post["id"] + new_parent_id = new_root_id + + self._insert_post_mapping(db_post["id"], new_parent_id) + + for uri, cid in created_records[1:]: + self._insert_post( + { + "user": self.did, + "service": self.url, + "identifier": uri, + "parent": new_parent_id, + "root": new_root_id, + "reposted": None, + "extra_data": json.dumps({"cid": cid}), + "crossposted": 1, + } + ) + reply_post = self._get_post(self.url, self.did, uri) + if not reply_post: + raise ValueError("Inserted reply post not found!") + new_parent_id = reply_post["id"] + self._insert_post_mapping(db_post["id"], new_parent_id) + + self.log.info( + "Post accepted successfully: %s -> %s", + post.id, + [r[0] for r in created_records], + ) + + @override + def delete_post(self, post: PostRef): + self.log.info( + "Deleting post %s (author: %s, service: %s)...", + post.id, + post.author, + post.service, + ) + db_post = self._get_post(post.service, post.author, post.id) + if not db_post: + self.log.warning( + "Post not found in DB: %s (author: %s, service: %s)", + post.id, + post.author, + post.service, + ) + return + + mappings = self._get_mappings(db_post["id"], self.url, self.did) + for mapping in mappings[::-1]: + self.log.info("Deleting '%s'...", mapping["identifier"]) + self._client.delete_post(mapping["identifier"]) + self._delete_post_by_id(mapping["id"]) + self.log.info("Post deleted successfully: %s", post.id) + + @override + def accept_repost(self, repost: PostRef, reposted: PostRef): + self.log.info( + "Accepting repost %s of %s (author: %s, service: %s)...", + repost.id, + reposted.id, + repost.author, + repost.service, + ) + db_repost = self._get_post(repost.service, repost.author, repost.id) + db_reposted = self._get_post(reposted.service, reposted.author, reposted.id) + if not db_repost or not db_reposted: + self.log.info("Post not found in db, skipping repost..") + return + + mappings = self._get_mappings(db_reposted["id"], self.url, self.did) + if not mappings: + return + + try: + cid = json.loads(mappings[0]["extra_data"])["cid"] + except (json.JSONDecodeError, AttributeError, KeyError): + self.log.exception("Failed to parse CID from extra_data!") + return + + response = self._client.repost(mappings[0]["identifier"], cid) + + self._insert_post( + { + "user": self.did, + "service": self.url, + "identifier": response.uri, + "parent": None, + "root": None, + "reposted": mappings[0]["id"], + "extra_data": json.dumps({"cid": response.cid}), + "crossposted": 1, + } + ) + inserted = self._get_post(self.url, self.did, response.uri) + if not inserted: + raise ValueError("Inserted post not found!") + self._insert_post_mapping(db_repost["id"], inserted["id"]) + self.log.info("Repost accepted successfully: %s", repost.id) + + @override + def delete_repost(self, repost: PostRef): + self.log.info( + "Deleting repost %s (author: %s, service: %s)...", + repost.id, + repost.author, + repost.service, + ) + db_repost = self._get_post(repost.service, repost.author, repost.id) + if not db_repost: + self.log.warning( + "Repost not found in DB: %s (author: %s, service: %s)", + repost.id, + repost.author, + repost.service, + ) + return + + mappings = self._get_mappings(db_repost["id"], self.url, self.did) + if mappings: + self.log.info("Deleting '%s'...", mappings[0]["identifier"]) + self._client.delete_repost(mappings[0]["identifier"]) + self._delete_post_by_id(mappings[0]["id"]) + self.log.info("Repost deleted successfully: %s", repost.id) + else: + self.log.error([mappings]) diff --git a/bluesky/richtext.py b/bluesky/richtext.py new file mode 100644 --- /dev/null +++ b/bluesky/richtext.py @@ -0,0 +1,171 @@ +from atproto.models import ( + Facet, + FacetFeature, + FacetIndex, + LinkFeature, + MentionFeature, + TagFeature, +) +from cross.tokens import LinkToken, MentionToken, TagToken, TextToken, Token +from util.splitter import canonical_label + + +def richtext_to_tokens(text: str, facets: list[dict]) -> list[Token]: + if not text: + return [] + ut8_text = text.encode("utf-8") + if not facets: + return [TextToken(text=ut8_text.decode("utf-8"))] + + slices: list[tuple[int, int, str, str]] = [] + for facet in facets: + features: list[dict] = facet.get("features", []) + if not features: + continue + feature = features[0] + feature_type = feature["$type"] + index = facet["index"] + match feature_type: + case "app.bsky.richtext.facet#tag": + slices.append( + (index["byteStart"], index["byteEnd"], "tag", feature["tag"]) + ) + case "app.bsky.richtext.facet#link": + slices.append( + (index["byteStart"], index["byteEnd"], "link", feature["uri"]) + ) + case "app.bsky.richtext.facet#mention": + slices.append( + (index["byteStart"], index["byteEnd"], "mention", feature["did"]) + ) + + if not slices: + return [TextToken(text=ut8_text.decode("utf-8"))] + + slices.sort(key=lambda s: s[0]) + unique: list[tuple[int, int, str, str]] = [] + current_end = 0 + for start, end, ttype, val in slices: + if start >= current_end: + unique.append((start, end, ttype, val)) + current_end = end + + if not unique: + return [TextToken(text=ut8_text.decode("utf-8"))] + + tokens: list[Token] = [] + prev = 0 + + for start, end, ttype, val in unique: + if start > prev: + tokens.append(TextToken(text=ut8_text[prev:start].decode("utf-8"))) + match ttype: + case "link": + label = ut8_text[start:end].decode("utf-8") + split = val.split("://", 1) + if ( + len(split) > 1 + and split[1].startswith(label) + or (label.endswith("...") and split[1].startswith(label[:-3])) + ): + tokens.append(LinkToken(href=val)) + prev = end + continue + tokens.append(LinkToken(href=val, label=label)) + case "tag": + tag = ut8_text[start:end].decode("utf-8") + tokens.append(TagToken(tag=tag[1:] if tag.startswith("#") else tag)) + case "mention": + mention = ut8_text[start:end].decode("utf-8") + tokens.append( + MentionToken( + username=mention[1:] if mention.startswith("@") else mention, + uri=val, + ) + ) + prev = end + + if prev < len(ut8_text): + tokens.append(TextToken(text=ut8_text[prev:].decode("utf-8"))) + + return tokens + + +def tokens_to_richtext(tokens: list[Token]) -> tuple[str, list[Facet]] | None: + segments: list[tuple[str, FacetFeature | None]] = [] + byte_offset = 0 + + for token in tokens: + match token: + case TextToken(): + text_bytes = token.text.encode("utf-8") + segments.append((token.text, None)) + byte_offset += len(text_bytes) + + case TagToken(): + tag_text = f"#{token.tag}" + tag_bytes = tag_text.encode("utf-8") + segments.append( + ( + tag_text, + TagFeature(tag=token.tag), + ) + ) + byte_offset += len(tag_bytes) + + case MentionToken(): + mention_text = f"@{token.username}" + mention_bytes = mention_text.encode("utf-8") + segments.append( + ( + mention_text, + MentionFeature(did=token.uri) + if token.uri + else MentionFeature(did=""), + ) + ) + byte_offset += len(mention_bytes) + + case LinkToken(): + href = token.href + label = token.label if token.label else href + + if canonical_label(token.label, token.href): + max_label_len = 30 + label_bytes = label.encode("utf-8") + if len(label_bytes) > max_label_len: + label = label[: max_label_len - 1] + "…" + label_bytes = label.encode("utf-8") + else: + label_bytes = label.encode("utf-8") + + segments.append( + ( + label, + LinkFeature(uri=href), + ) + ) + byte_offset += len(label_bytes) + + case _: + return None + + text = "".join(seg[0] for seg in segments) + facets: list[Facet] = [] + + current_offset = 0 + for seg_text, seg_feature in segments: + if seg_feature: + seg_bytes = seg_text.encode("utf-8") + facets.append( + Facet( + index=FacetIndex( + byte_start=current_offset, + byte_end=current_offset + len(seg_bytes), + ), + features=[seg_feature], + ) + ) + current_offset += len(seg_text.encode("utf-8")) + + return text, facets diff --git a/bluesky/tokens.py b/bluesky/tokens.py new file mode 100644 --- /dev/null +++ b/bluesky/tokens.py @@ -0,0 +1,95 @@ +from cross.tokens import LinkToken, MentionToken, TagToken, TextToken, Token + + +def tokenize_post(text: str, facets: list[dict]) -> list[Token]: + def decode(ut8: bytes) -> str: + return ut8.decode(encoding="utf-8") + + if not text: + return [] + ut8_text = text.encode(encoding="utf-8") + if not facets: + return [TextToken(text=decode(ut8_text))] + + slices: list[tuple[int, int, str, str]] = [] + + for facet in facets: + features: list[dict] = facet.get("features", []) + if not features: + continue + + # we don't support overlapping facets/features + feature = features[0] + feature_type = feature["$type"] + index = facet["index"] + match feature_type: + case "app.bsky.richtext.facet#tag": + slices.append( + (index["byteStart"], index["byteEnd"], "tag", feature["tag"]) + ) + case "app.bsky.richtext.facet#link": + slices.append( + (index["byteStart"], index["byteEnd"], "link", feature["uri"]) + ) + case "app.bsky.richtext.facet#mention": + slices.append( + (index["byteStart"], index["byteEnd"], "mention", feature["did"]) + ) + + if not slices: + return [TextToken(text=decode(ut8_text))] + + slices.sort(key=lambda s: s[0]) + unique: list[tuple[int, int, str, str]] = [] + current_end = 0 + for start, end, ttype, val in slices: + if start >= current_end: + unique.append((start, end, ttype, val)) + current_end = end + + if not unique: + return [TextToken(text=decode(ut8_text))] + + tokens: list[Token] = [] + prev = 0 + + for start, end, ttype, val in unique: + if start > prev: + # text between facets + tokens.append(TextToken(text=decode(ut8_text[prev:start]))) + # facet token + match ttype: + case "link": + label = decode(ut8_text[start:end]) + + # try to unflatten links + split = val.split("://", 1) + if len(split) > 1: + if split[1].startswith(label): + tokens.append(LinkToken(href=val)) + prev = end + continue + + if label.endswith("...") and split[1].startswith(label[:-3]): + tokens.append(LinkToken(href=val)) + prev = end + continue + + tokens.append(LinkToken(href=val, label=label)) + case "tag": + tag = decode(ut8_text[start:end]) + tokens.append(TagToken(tag=tag[1:] if tag.startswith("#") else tag)) + case "mention": + mention = decode(ut8_text[start:end]) + tokens.append( + MentionToken( + username=mention[1:] if mention.startswith("@") else mention, + uri=val, + ) + ) + prev = end + + if prev < len(ut8_text): + tokens.append(TextToken(text=decode(ut8_text[prev:]))) + + return tokens diff --git a/cross/__init__.py b/cross/__init__.py new file mode 100644 --- /dev/null +++ b/cross/__init__.py diff --git a/cross/attachments.py b/cross/attachments.py new file mode 100644 --- /dev/null +++ b/cross/attachments.py @@ -0,0 +1,39 @@ +from dataclasses import dataclass + +from cross.media import Blob + + +@dataclass(kw_only=True) +class Attachment: + pass + + +@dataclass(kw_only=True) +class LabelsAttachment(Attachment): + labels: list[str] + + +@dataclass(kw_only=True) +class LanguagesAttachment(Attachment): + langs: list[str] + + +@dataclass(kw_only=True) +class SensitiveAttachment(Attachment): + sensitive: bool + + +@dataclass(kw_only=True) +class RemoteUrlAttachment(Attachment): + url: str + + +@dataclass(kw_only=True) +class MediaAttachment(Attachment): + blobs: list[Blob] + + +@dataclass(kw_only=True) +class QuoteAttachment(Attachment): + quoted_id: str + quoted_user: str diff --git a/cross/media.py b/cross/media.py new file mode 100644 --- /dev/null +++ b/cross/media.py @@ -0,0 +1,176 @@ +import json +import os +import re +import subprocess +import urllib.parse +from dataclasses import dataclass, field +from typing import Any, cast + +import httpx +import magic + + +FILENAME = re.compile(r'filename="?([^\";]*)"?') +MAGIC = magic.Magic(mime=True) + + +@dataclass +class Blob: + url: str + mime: str + io: bytes = field(repr=False) + name: str | None = None + alt: str | None = None + + +@dataclass +class MediaInfo: + width: int + height: int + duration: float | None = None + + +def mime_from_bytes(io: bytes) -> str: + mime = MAGIC.from_buffer(io) + if not mime: + mime = "application/octet-stream" + return str(mime) + + +def download_blob( + url: str, + alt: str | None = None, + max_bytes: int = 100_000_000, + client: httpx.Client | None = None, +) -> Blob | None: + name = get_filename_from_url(url, client) + io = download_chuncked(url, max_bytes, client) + if not io: + return None + return Blob(url, mime_from_bytes(io), io, name, alt) + + +def download_chuncked( + url: str, max_bytes: int = 100_000_000, client: httpx.Client | None = None +) -> bytes | None: + if client is None: + client = httpx.Client() + with client.stream("GET", url, timeout=20) as response: + if response.status_code != 200: + return None + + downloaded_bytes = b"" + current_size = 0 + + for chunk in response.iter_bytes(chunk_size=8192): + if not chunk: + continue + + current_size += len(chunk) + if current_size > max_bytes: + return None + + downloaded_bytes += chunk + + return downloaded_bytes + + +def get_filename_from_url(url: str, client: httpx.Client | None = None) -> str: + try: + if client is None: + client = httpx.Client() + response = client.head(url, timeout=5, follow_redirects=True) + disposition = response.headers.get("Content-Disposition") + if disposition: + filename = FILENAME.findall(disposition) + if filename: + return str(filename[0]) + except httpx.RequestError: + pass + + parsed_url = urllib.parse.urlparse(url) + base_name = os.path.basename(parsed_url.path) + + # hardcoded fix to return the cid for pds blobs + if base_name == "com.atproto.sync.getBlob": + qs = urllib.parse.parse_qs(parsed_url.query) + if qs and qs.get("cid"): + return str(qs["cid"][0]) + + return base_name + + +def convert_to_mp4(video: Blob) -> Blob: + cmd = [ + "ffmpeg", + "-i", "pipe:0", + "-c:v", "copy", + "-c:a", "aac", + "-b:a", "128k", + "-movflags", "frag_keyframe+empty_moov+default_base_moof", + "-f", "mp4", + "pipe:1", + ] # fmt: skip + + proc = subprocess.Popen( + cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + out_bytes, err = proc.communicate(input=video.io) + + if proc.returncode != 0: + raise RuntimeError(f"ffmpeg compress failed: {err.decode()}") + + return Blob(video.url, mime_from_bytes(out_bytes), out_bytes, video.name, video.alt) + + +def compress_image(image: Blob, quality: int = 95) -> Blob: + cmd = [ + "ffmpeg", + "-f", "image2pipe", + "-i", "pipe:0", + "-c:v", "webp", + "-q:v", str(quality), + "-f", "image2pipe", + "pipe:1", + ] # fmt: skip + + proc = subprocess.Popen( + cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + out_bytes, err = proc.communicate(input=image.io) + + if proc.returncode != 0: + raise RuntimeError(f"ffmpeg compress failed: {err.decode()}") + + return Blob(image.url, "image/webp", out_bytes, image.name, image.alt) + + +def probe_bytes(bytes: bytes) -> dict[str, Any]: + cmd = [ + "ffprobe", + "-v", "error", + "-show_format", + "-show_streams", + "-print_format", "json", + "pipe:0", + ] # fmt: skip + proc = subprocess.run(cmd, input=bytes, capture_output=True) + + if proc.returncode != 0: + raise RuntimeError(f"ffprobe failed: {proc.stderr.decode()}") + + return json.loads(proc.stdout) # type: ignore[no-any-return] + + +def get_media_meta(bytes: bytes) -> MediaInfo: + probe = probe_bytes(bytes) + streams = [s for s in probe["streams"] if s["codec_type"] == "video"] + if not streams: + raise ValueError("No video stream found") + + media: dict[str, Any] = cast(dict[str, Any], streams[0]) + return MediaInfo( + width=int(media["width"]), + height=int(media["height"]), + duration=float(media.get("duration", probe["format"].get("duration"))), + ) diff --git a/cross/post.py b/cross/post.py new file mode 100644 --- /dev/null +++ b/cross/post.py @@ -0,0 +1,42 @@ +from dataclasses import dataclass, field +from typing import TypeVar + +from cross.attachments import Attachment +from cross.tokens import Token + + +T = TypeVar("T", bound=Attachment) + + +class AttachmentKeeper: + def __init__(self) -> None: + self._map: dict[type, Attachment] = {} + + def put(self, attachment: Attachment) -> None: + self._map[attachment.__class__] = attachment + + def get(self, cls: type[T]) -> T | None: + instance = self._map.get(cls) + if instance is None: + return None + if not isinstance(instance, cls): + raise TypeError(f"Expected {cls.__name__}, got {type(instance).__name__}") + return instance + + def __repr__(self) -> str: + return f"AttachmentKeeper(_map={self._map.values()})" + + +@dataclass(kw_only=True) +class PostRef: + id: str + author: str + service: str + + +@dataclass(kw_only=True) +class Post(PostRef): + parent_id: str | None + tokens: list[Token] + text_type: str = "text/plain" + attachments: AttachmentKeeper = field(default_factory=AttachmentKeeper) diff --git a/cross/service.py b/cross/service.py new file mode 100644 --- /dev/null +++ b/cross/service.py @@ -0,0 +1,175 @@ +import logging +import sqlite3 +from abc import ABC, abstractmethod +from collections.abc import Callable +from typing import Any, cast + +from cross.post import Post, PostRef +from database.connection import DatabasePool + + +columns: list[str] = [ + "user", + "service", + "identifier", + "parent", + "root", + "reposted", + "extra_data", + "crossposted", +] +placeholders: str = ", ".join(["?" for _ in columns]) +column_names: str = ", ".join(columns) + + +class Service: + def __init__(self, url: str, db: DatabasePool) -> None: + self.url: str = url + self.db: DatabasePool = db + self.log: logging.Logger = logging.getLogger(self.__class__.__name__) + # self._lock: threading.Lock = threading.Lock() + + def _get_post(self, url: str, user: str, identifier: str) -> sqlite3.Row | None: + cursor = self.db.get_conn().cursor() + _ = cursor.execute( + """ + SELECT * FROM posts + WHERE service = ? + AND user = ? + AND identifier = ? + """, + (url, user, identifier), + ) + return cast(sqlite3.Row, cursor.fetchone()) + + def _get_post_by_id(self, id: int) -> sqlite3.Row | None: + cursor = self.db.get_conn().cursor() + _ = cursor.execute("SELECT * FROM posts WHERE id = ?", (id,)) + return cast(sqlite3.Row, cursor.fetchone()) + + def _get_mappings( + self, original: int, service: str, user: str + ) -> list[sqlite3.Row]: + cursor = self.db.get_conn().cursor() + _ = cursor.execute( + """ + SELECT * + FROM posts AS p + JOIN mappings AS m + ON p.id = m.mapped + WHERE m.original = ? + AND p.service = ? + AND p.user = ? + ORDER BY p.id; + """, + (original, service, user), + ) + return cursor.fetchall() + + def _find_mapped_thread( + self, parent: str, iservice: str, iuser: str, oservice: str, ouser: str + ): + reply_data = self._get_post(iservice, iuser, parent) + if not reply_data: + return None + + reply_mappings: list[sqlite3.Row] | None = self._get_mappings( + reply_data["id"], oservice, ouser + ) + if not reply_mappings: + return None + + reply_identifier: sqlite3.Row = reply_mappings[-1] + root_identifier: sqlite3.Row = reply_mappings[0] + + if reply_data["root"]: + root_data = self._get_post_by_id(reply_data["root"]) + if not root_data: + return None + + root_mappings = self._get_mappings(reply_data["root"], oservice, ouser) + if not root_mappings: + return None + root_identifier = root_mappings[0] + + return ( + root_identifier["identifier"], # real ids + reply_identifier["identifier"], + reply_data["root"], # db ids + reply_data["id"], + ) + + def _insert_post(self, post_data: dict[str, Any]): + values = [post_data.get(col) for col in columns] + cursor = self.db.get_conn().cursor() + _ = cursor.execute( + f"INSERT INTO posts ({column_names}) VALUES ({placeholders})", values + ) + + def _insert_post_mapping(self, original: int, mapped: int): + cursor = self.db.get_conn().cursor() + _ = cursor.execute( + "INSERT OR IGNORE INTO mappings (original, mapped) VALUES (?, ?);", + (original, mapped), + ) + _ = cursor.execute( + "INSERT OR IGNORE INTO mappings (original, mapped) VALUES (?, ?);", + (mapped, original), + ) + + def _delete_post(self, url: str, user: str, identifier: str): + cursor = self.db.get_conn().cursor() + _ = cursor.execute( + """ + DELETE FROM posts + WHERE identifier = ? + AND service = ? + AND user = ? + """, + (identifier, url, user), + ) + + def _delete_post_by_id(self, id: int): + cursor = self.db.get_conn().cursor() + _ = cursor.execute("DELETE FROM posts WHERE id = ?", (id,)) + + def _is_post_crossposted(self, url: str, user: str, identifier: str) -> bool: + cursor = self.db.get_conn().cursor() + row = cursor.execute( + """ + SELECT crossposted FROM posts + WHERE service = ? + AND user = ? + AND identifier = ? + """, + (url, user, identifier), + ).fetchone() + return bool(row and row["crossposted"]) + + +class OutputService(Service): + def accept_post(self, post: Post): + self.log.warning("NOT IMPLEMENTED (%s), accept_post %s", self.url, post.id) + + def delete_post(self, post: PostRef): + self.log.warning("NOT IMPLEMENTED (%s), delete_post %s", self.url, post.id) + + def accept_repost(self, repost: PostRef, reposted: PostRef): + self.log.warning( + "NOT IMPLEMENTED (%s), accept_repost %s of %s", + self.url, + repost.id, + reposted.id, + ) + + def delete_repost(self, repost: PostRef): + self.log.warning("NOT IMPLEMENTED (%s), delete_repost %s", self.url, repost.id) + + +class InputService(ABC, Service): + outputs: list[OutputService] + submitter: Callable[[Callable[[], None]], None] + + @abstractmethod + async def listen(self): + pass diff --git a/cross/tokens.py b/cross/tokens.py new file mode 100644 --- /dev/null +++ b/cross/tokens.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass + + +@dataclass(kw_only=True) +class Token: + pass + + +@dataclass(kw_only=True) +class TextToken(Token): + text: str + + +@dataclass(kw_only=True) +class LinkToken(Token): + href: str + label: str | None = None + + +@dataclass(kw_only=True) +class TagToken(Token): + tag: str + + +@dataclass(kw_only=True) +class MentionToken(Token): + username: str + uri: str | None = None diff --git a/database/__init__.py b/database/__init__.py new file mode 100644 --- /dev/null +++ b/database/__init__.py diff --git a/database/connection.py b/database/connection.py new file mode 100644 --- /dev/null +++ b/database/connection.py @@ -0,0 +1,33 @@ +import sqlite3 +import threading +from pathlib import Path + + +class DatabasePool: + def __init__(self, db: Path) -> None: + self.db: Path = db + self._local: threading.local = threading.local() + self._conns: list[sqlite3.Connection] = [] + + def get_conn(self) -> sqlite3.Connection: + if getattr(self._local, "conn", None) is None: + self._local.conn = get_conn(self.db) + self._conns.append(self._local.conn) + return self._local.conn # type: ignore[no-any-return] + + def close(self) -> None: + for c in self._conns: + c.close() + + +def get_conn(db: Path) -> sqlite3.Connection: + conn = sqlite3.connect(db, autocommit=True, check_same_thread=False) + conn.row_factory = sqlite3.Row + _ = conn.executescript(""" + PRAGMA journal_mode = WAL; + PRAGMA mmap_size = 134217728; + PRAGMA cache_size = 4000; + PRAGMA synchronous = NORMAL; + PRAGMA foreign_keys = ON; + """) + return conn diff --git a/database/migrations.py b/database/migrations.py new file mode 100644 --- /dev/null +++ b/database/migrations.py @@ -0,0 +1,61 @@ +import sqlite3 +from collections.abc import Callable +from pathlib import Path + +from database.connection import get_conn +from util.util import LOGGER + + +class DatabaseMigrator: + def __init__(self, db_path: Path, migrations_folder: Path) -> None: + self.db_path: Path = db_path + self.migrations_folder: Path = migrations_folder + self.conn: sqlite3.Connection = get_conn(db_path) + _ = self.conn.execute("PRAGMA foreign_keys = OFF;") + self.conn.autocommit = False + + def close(self): + self.conn.close() + + def get_version(self) -> int: + cursor = self.conn.cursor() + _ = cursor.execute("PRAGMA user_version") + return int(cursor.fetchone()[0]) + + def set_version(self, version: int): + cursor = self.conn.cursor() + _ = cursor.execute(f"PRAGMA user_version = {version}") + self.conn.commit() + + def apply_migration( + self, + version: int, + filename: str, + migration: Callable[[sqlite3.Connection], None], + ) -> None: + try: + migration(self.conn) + self.set_version(version) + self.conn.commit() + LOGGER.info("Applied migration: %s..", filename) + except sqlite3.Error as e: + self.conn.rollback() + raise Exception(f"Error applying migration {filename}: {e}") + + def migrate(self): + current_version = self.get_version() + from migrations._registry import load_migrations + + migrations = load_migrations(self.migrations_folder) + + if not migrations: + LOGGER.warning("No migration files found.") + return + + pending = [m for m in migrations if m[0] > current_version] + if not pending: + LOGGER.info("No pending migrations.") + return + + for version, filename, migration in pending: + self.apply_migration(version, filename, migration) diff --git a/docs/README.md b/docs/README.md new file mode 100644 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,91 @@ +# XPost documentation + +## Installation + +The recommended approach is to use the official container image from `ghcr.io/zenfyrdev/xpost:latest` + +### Podman Quadlets + +Example Rootful Quadlet. Make sure the data dir exists on the host and is owned by `1000:1000`! + +``` +[Unit] +Description=XPost + +[Container] +Image=ghcr.io/zenfyrdev/xpost:latest +EnvironmentFile=/etc/containers/systemd/xpost/.env +Volume=/var/containers/xpost/data:/app/data:Z + +[Service] +Restart=always +RestartSec=10s + +[Install] +WantedBy=default.target +``` + +### Docker Compose + +Make sure the data dir exists on the host and is owned by `1000:1000`! + +``` +services: + xpost: + image: ghcr.io/zenfyrdev/xpost:latest + restart: unless-stopped + env_file: ./.env + volumes: + - ./data:/app/data +``` + +### Native Install + +The project uses uv, so a native install using a `.venv` is pretty simple. + +1. Make sure that `ffmpeg` and `libmagic` are installed. +2. Download and install [uv](https://github.com/astral-sh/uv) +3. Clone the project `https://tangled.org/zenfyr.dev/xpost` +4. Run `uv sync --locked` + + +## Quickstart + +Upon first launch, XPost will create a folder `./data` with an example `settings.json` + +Edit the file, and then start XPost again. + +## Features + +- Based on streaming APIs to instantly crosspost new posts. +- Splitting large posts into multiple smaller ones to fit into character limits. +- Supports quotes and reposts. + + +## Configuration + +XPost config accepts an array of input -> outputs pairs. find all services and example configs in [services.md](./services.md). + +All `"key": "value"` options can be set to `env:VARIABLE` to read envvars instead of storing things directly in the settings file. + +Additionally XPost accepts some envvars. + +| Key | Description | +|-----------------|---------------------------------------------------------------------------------------------------------------| +| `DATA_DIR` | Base data directory. | +| `SETTINGS_DIR` | `settings.json` file location. | +| `DATABASE_DIR` | `data.db` file location. | +| `SLINGSHOT_URL` | URL of the [microcosm](https://www.microcosm.blue/) slingshot service for resolving identities. | +| `JETSTREAM_URL` | URL of the [Jetstream](https://github.com/bluesky-social/jetstream) service for listening for incoming posts. | + +## Advanced Features + +### bi-directional crossposting + +**This is experimental and unstable.** + +XPost supports pointing to services directly at each other, posts crossposted from other services are marked as so, and are skipped to avoid creating infinite loops. + +While this works for most things, cross-service replies can end up being duplicated multiple times over or cause an infinite loop, this usually happens when a post from one service is split into multiple other on the other. + +e.g. Post A from Mastodon ends up as Post A1 and Post A2 on bsky, replying from bsky to either may result in Reply B being posted twice, or getting stuck in a loop. diff --git a/docs/services.md b/docs/services.md new file mode 100644 --- /dev/null +++ b/docs/services.md @@ -0,0 +1,162 @@ +# Services + +## Input + +Input services ingest data from other websites. + +### Bluesky Jetstream + +This service uses a [Jetstream](https://github.com/bluesky-social/jetstream) to listen for posts. + +``` +{ + "services": [ + { + "input": { + "type": "bluesky-jetstream", + "handle": "bsky.app" + }, + "outputs": [] + } + ] +} +``` + +| Key | Description | +|----------|-----------------------------------------------------------------------| +| `handle` | Account handle. Used to resolve `did` and `pds`. | +| `did` | Account identifier. Can be specified instead of a `handle`. | +| `pds` | Account host. Optional, will be resolved from `did` if not specified. | + +### Mastodon WebSocket + +Uses a WebSocket to listen to the home timeline. + +``` +{ + "services": [ + { + "input": { + "type": "mastodon-wss", + "instance": "https://mastodon.social", + "token": "***" + }, + "outputs": [] + } + ] +} +``` + +| Key | Description | +|----------------------|--------------------------------------------------------| +| `instance` | Account host. | +| `token` | Account access token. | +| `allowed_visibility` | Post visibilities that ware allowed to be crossposted. | + +#### Getting a token + +**Mastodon:** + +- Go to Settings -> Development +- Click "New Application" +- Set a name (e.g. xpost), allow "read", "write", "profile" perms. +- Click on the new application and copy "Your access token" + +**Non-Mastodon** + +Software like iceshrimp/akkoma can either use https://getauth.thms.uk/?client_name=xpost&scopes=read%20write%20profile or get the token using dev tools on any web client. (any `/api/v*` request, the `authorization` header, copy the value besides `Bearer `) + +### Misskey WebSocket + +Uses a WebSocket to listen to the home timeline channel. + +> [!NOTE] +> Misskey WSS doesn't support deletes, crossposted posts have to be manually deleted (or look into [bi-directional](./README.md#bi-directional-crossposting) crossposting) + +``` +{ + "services": [ + { + "input": { + "type": "misskey-wss", + "instance": "https://misskey.io", + "token": "***" + }, + "outputs": [] + } + ] +} +``` + +| Key | Description | +|----------------------|--------------------------------------------------------| +| `instance` | Account host. | +| `token` | Account access token. | +| `allowed_visibility` | Post visibilities that ware allowed to be crossposted. | + +#### Getting a token + +Use Dev Tools 💔 + +## Output + +### Bluesky + +``` +{ + "services": [ + { + "input": {}, + "outputs": [ + { + "type": "bluesky", + "handle": "bsky.app", + "password": "***" + } + ] + } + ] +} +``` + +| Key | Description | +|---------------|---------------------------------------------------------------------------------------------------------| +| `handle` | Account handle. Used to resolve `did` and `pds`. | +| `did` | Account identifier. Can be specified instead of a `handle`. | +| `pds` | Account host. Optional, will be resolved from `did` if not specified. | +| `password` | Account App Password. | +| `quote_gate` | Disable ability for others to quote. | +| `thread_gate` | Limit replies to the post. null - everybody, [] - nobody. accepts "mentioned", "following", "followers" | + +#### App Password + +Please do not use the main password. + +- Go to Settings -> Privacy and Security -> App Passwords +- Click "Add App Password" +- Copy the new password (it will not be shown again!) + +### Mastodon + +``` +{ + "services": [ + { + "input": {}, + "outputs": [ + { + "type": "mastodon", + "instance": "https://mastodon.social", + "token": "***" + } + ] + } + ] +} +``` + +| Key | Description | +|--------------|--------------------------------------------------------| +| `instance` | Account host. | +| `token` | Account access token. | +| `visibility` | What visibility to set for crossposted posts | \ No newline at end of file diff --git a/env.py b/env.py new file mode 100644 --- /dev/null +++ b/env.py @@ -0,0 +1,18 @@ +import os +from pathlib import Path + + +DEV = bool(os.environ.get("DEV")) or False + +DATA_DIR = Path(os.environ.get("DATA_DIR") or "./data") +SETTINGS_DIR = Path( + os.environ.get("SETTINGS_DIR") or DATA_DIR.joinpath("settings.json") +) +DATABASE_DIR = Path(os.environ.get("DATABASE_DIR") or DATA_DIR.joinpath("data.db")) + +MIGRATIONS_DIR = Path(os.environ.get("MIGRATIONS_DIR") or "./migrations") + +SLINGSHOT_URL = os.environ.get("SLINGSHOT_URL") or "https://slingshot.microcosm.blue" +JETSTREAM_URL = ( + os.environ.get("JETSTREAM_URL") or "wss://jetstream2.us-west.bsky.network/subscribe" +) diff --git a/main.py b/main.py new file mode 100644 --- /dev/null +++ b/main.py @@ -0,0 +1,166 @@ +import argparse +import asyncio +import json +import queue +import threading +from collections.abc import Callable +from typing import Any + +import env +from database.connection import DatabasePool +from database.migrations import DatabaseMigrator +from util.util import LOGGER, read_env, shutdown_hook + + +EXAMPLE_CONFIG = { + "services": [ + { + "input": {"type": "bluesky-jetstream", "handle": "bsky.app"}, + "outputs": [ + { + "type": "mastodon", + "instance": "https://mastodon.social", + "token": "env:MASTODON_TOKEN", + } + ], + } + ] +} + + +def dump_example_config() -> None: + env.SETTINGS_DIR.parent.mkdir(parents=True, exist_ok=True) + with open(env.SETTINGS_DIR, "w") as f: + json.dump(EXAMPLE_CONFIG, f, indent=2) + + +def flush_caches() -> None: + from atproto.store import flush_caches as flush_atproto_caches + from atproto.store import get_store + + db_pool = DatabasePool(env.DATABASE_DIR) + get_store(db_pool) + + LOGGER.info("Flushing atproto caches...") + sessions, identities = flush_atproto_caches() + LOGGER.info("Flushed %d sessions and %d identities", sessions, identities) + LOGGER.info("Cache flush complete!") + + db_pool.close() + + +def main() -> None: + parser = argparse.ArgumentParser( + description="xpost: social media crossposting tool" + ) + parser.add_argument( + "--flush-caches", + action="store_true", + help="Flush all caches (like sessions and identities)", + ) + args = parser.parse_args() + + if args.flush_caches: + flush_caches() + return + + if not env.DATA_DIR.exists(): + env.DATA_DIR.mkdir(parents=True) + + if not env.SETTINGS_DIR.exists(): + LOGGER.info("First launch detected! Creating %s and exiting!", env.SETTINGS_DIR) + dump_example_config() + LOGGER.info("Example config written to %s", env.SETTINGS_DIR) + LOGGER.info("Please edit the config file and run again!") + return + + migrator = DatabaseMigrator(env.DATABASE_DIR, env.MIGRATIONS_DIR) + try: + migrator.migrate() + except Exception: + LOGGER.exception("Failed to migrate database!") + return + finally: + migrator.close() + + db_pool = DatabasePool(env.DATABASE_DIR) + import httpx + + http_client = httpx.Client(timeout=httpx.Timeout(30)) + + LOGGER.info("Bootstrapping registries...") + from registry import create_input_service, create_output_service + from registry_bootstrap import bootstrap + + bootstrap() + + LOGGER.info("Loading settings...") + + with open(env.SETTINGS_DIR) as f: + settings = json.load(f) + read_env(settings) + + if "services" not in settings: + raise KeyError("No `services` specified in settings!") + + service_pairs: list[tuple[Any, list[Any]]] = [] + for svc in settings["services"]: + if "input" not in svc: + raise KeyError("Each service must have an `input` field!") + if "outputs" not in svc: + raise KeyError("Each service must have an `outputs` field!") + + inp = create_input_service(db_pool, http_client, svc["input"]) + outs = [ + create_output_service(db_pool, http_client, data) for data in svc["outputs"] + ] + service_pairs.append((inp, outs)) + + LOGGER.info("Starting task worker...") + + def worker(task_queue: queue.Queue[Callable[[], None] | None]): + while True: + task = task_queue.get() + if task is None: + break + + try: + task() + except Exception: + LOGGER.exception("Exception in worker thread!") + finally: + task_queue.task_done() + + task_queue: queue.Queue[Callable[[], None] | None] = queue.Queue() + thread = threading.Thread(target=worker, args=(task_queue,), daemon=True) + thread.start() + + for inp, outs in service_pairs: + inp.outputs = outs + inp.submitter = lambda c: task_queue.put(c) + + inputs = [inp for inp, _ in service_pairs] + LOGGER.info("Starting %d input service(s)...", len(inputs)) + try: + asyncio.run(_run_all_inputs(inputs)) + except KeyboardInterrupt: + LOGGER.info("Stopping...") + + task_queue.join() + task_queue.put(None) + thread.join() + + for shook in shutdown_hook: + shook() + + db_pool.close() + http_client.close() + + +async def _run_all_inputs(inputs: list[Any]) -> None: + tasks = [asyncio.create_task(inp.listen()) for inp in inputs] + await asyncio.gather(*tasks, return_exceptions=True) + + +if __name__ == "__main__": + main() diff --git a/mastodon/__init__.py b/mastodon/__init__.py new file mode 100644 --- /dev/null +++ b/mastodon/__init__.py diff --git a/mastodon/info.py b/mastodon/info.py new file mode 100644 --- /dev/null +++ b/mastodon/info.py @@ -0,0 +1,143 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any, override + +import httpx + +from cross.service import Service +from cross.tokens import LinkToken, MentionToken, TagToken +from database.connection import DatabasePool +from util.html import HTMLToTokensParser +from util.util import normalize_service_url + + +def validate_and_transform(data: dict[str, Any]): + if "token" not in data or "instance" not in data: + raise KeyError("Missing required values 'token' or 'instance'") + + data["instance"] = normalize_service_url(data["instance"]) + + +@dataclass(kw_only=True) +class InstanceInfo: + max_characters: int = 500 + max_media_attachments: int = 4 + characters_reserved_per_url: int = 23 + + image_size_limit: int = 16777216 + video_size_limit: int = 103809024 + + text_format: str = "text/plain" + + @classmethod + def from_api(cls, data: dict[str, Any]) -> "InstanceInfo": + config: dict[str, Any] = {} + + if "statuses" in data: + statuses_config: dict[str, Any] = data.get("statuses", {}) + if "max_characters" in statuses_config: + config["max_characters"] = statuses_config["max_characters"] + if "max_media_attachments" in statuses_config: + config["max_media_attachments"] = statuses_config[ + "max_media_attachments" + ] + if "characters_reserved_per_url" in statuses_config: + config["characters_reserved_per_url"] = statuses_config[ + "characters_reserved_per_url" + ] + + # glitch content type + if "supported_mime_types" in statuses_config: + text_mimes: list[str] = statuses_config["supported_mime_types"] + + if "text/x.misskeymarkdown" in text_mimes: + config["text_format"] = "text/x.misskeymarkdown" + elif "text/markdown" in text_mimes: + config["text_format"] = "text/markdown" + + if "media_attachments" in data: + media_config: dict[str, Any] = data["media_attachments"] + if "image_size_limit" in media_config: + config["image_size_limit"] = media_config["image_size_limit"] + if "video_size_limit" in media_config: + config["video_size_limit"] = media_config["video_size_limit"] + + # *oma extensions + if "max_toot_chars" in data: + config["max_characters"] = data["max_toot_chars"] + if "upload_limit" in data: + config["image_size_limit"] = data["upload_limit"] + config["video_size_limit"] = data["upload_limit"] + + if "pleroma" in data: + pleroma: dict[str, Any] = data["pleroma"] + if "metadata" in pleroma: + metadata: dict[str, Any] = pleroma["metadata"] + if "post_formats" in metadata: + post_formats: list[str] = metadata["post_formats"] + + if "text/x.misskeymarkdown" in post_formats: + config["text_format"] = "text/x.misskeymarkdown" + elif "text/markdown" in post_formats: + config["text_format"] = "text/markdown" + + return InstanceInfo(**config) + + +class MastodonService(ABC, Service): + def __init__(self, url: str, db: DatabasePool, http: httpx.Client) -> None: + super().__init__(url, db) + self.http = http + + def verify_credentials(self): + token = self._get_token() + response = self.http.get( + f"{self.url}/api/v1/accounts/verify_credentials", + headers={"Authorization": f"Bearer {token}"}, + ) + if response.status_code != 200: + self.log.error("Failed to validate user credentials!") + response.raise_for_status() + return dict(response.json()) + + def fetch_instance_info(self): + token = self._get_token() + responce = self.http.get( + f"{self.url}/api/v1/instance", + headers={"Authorization": f"Bearer {token}"}, + ) + if responce.status_code != 200: + self.log.error("Failed to get instance info!") + responce.raise_for_status() + return dict(responce.json()) + + @abstractmethod + def _get_token(self) -> str: + pass + + +class StatusParser(HTMLToTokensParser): + def __init__(self, status: dict[str, Any]) -> None: + super().__init__() + self.tags: set[str] = {tag["url"] for tag in status.get("tags", [])} + self.mentions: set[str] = {m["url"] for m in status.get("mentions", [])} + + @override + def handle_a_endtag(self): + label, _attr = self._tag_stack.pop("a") + + href = _attr.get("href") + if href: + cls = _attr.get("class", "") + if cls: + if "hashtag" in cls and href in self.tags: + tag = label[1:] if label.startswith("#") else label + + self.tokens.append(TagToken(tag=tag)) + return + if "mention" in cls and href in self.mentions: + username = label[1:] if label.startswith("@") else label + + self.tokens.append(MentionToken(username=username, uri=href)) + return + self.tokens.append(LinkToken(href=href, label=label)) diff --git a/mastodon/input.py b/mastodon/input.py new file mode 100644 --- /dev/null +++ b/mastodon/input.py @@ -0,0 +1,281 @@ +import asyncio +import json +import re +from dataclasses import dataclass, field +from typing import Any, cast, override + +import httpx +import websockets + +from cross.attachments import ( + LabelsAttachment, + LanguagesAttachment, + MediaAttachment, + QuoteAttachment, + RemoteUrlAttachment, + SensitiveAttachment, +) +from cross.media import Blob, download_blob +from cross.post import Post, PostRef +from cross.service import InputService +from database.connection import DatabasePool +from mastodon.info import MastodonService, StatusParser, validate_and_transform + + +ALLOWED_VISIBILITY: list[str] = ["public", "unlisted"] + + +@dataclass(kw_only=True) +class MastodonInputOptions: + token: str + instance: str + allowed_visibility: list[str] = field( + default_factory=lambda: ALLOWED_VISIBILITY.copy() + ) + filters: list[re.Pattern[str]] = field(default_factory=lambda: []) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "MastodonInputOptions": + validate_and_transform(data) + + if "allowed_visibility" in data: + for vis in data.get("allowed_visibility", []): + if vis not in ALLOWED_VISIBILITY: + raise ValueError(f"Invalid visibility option {vis}!") + + if "filters" in data: + data["filters"] = [re.compile(r) for r in data["filters"]] + + return MastodonInputOptions(**data) + + +class MastodonInputService(MastodonService, InputService): + def __init__( + self, db: DatabasePool, http: httpx.Client, options: MastodonInputOptions + ) -> None: + super().__init__(options.instance, db, http) + self.options: MastodonInputOptions = options + + self.log.info("Verifying %s credentails...", self.url) + response = self.verify_credentials() + self.user_id: str = response["id"] + + self.log.info("Getting %s configuration...", self.url) + response = self.fetch_instance_info() + self.streaming_url: str = response["urls"]["streaming_api"] + + @override + def _get_token(self) -> str: + return self.options.token + + def _on_create_post(self, status: dict[str, Any]): + self.log.info("Processing new post: %s", status["id"]) + + if status["account"]["id"] != self.user_id: + return + + if status["visibility"] not in self.options.allowed_visibility: + self.log.info( + "Skipping post with disallowed visibility: %s (%s)", + status["id"], + status["visibility"], + ) + return + + if self._is_post_crossposted(self.url, self.user_id, status["id"]): + self.log.info( + "Skipping %s, already crossposted", + status["id"], + ) + return + + reblog: dict[str, Any] | None = status.get("reblog") + if reblog: + if reblog["account"]["id"] != self.user_id: + return + self._on_reblog(status, reblog) + return + + if status.get("poll"): + self.log.info("Skipping '%s'! Contains a poll..", status["id"]) + return + + quote: dict[str, Any] | None = status.get("quote") + if quote: + quote = quote["quoted_status"] if quote.get("quoted_status") else quote + if not quote or quote["account"]["id"] != self.user_id: + return + + rquote = self._get_post(self.url, self.user_id, quote["id"]) + if not rquote: + self.log.info( + "Skipping %s, parent %s not found in db", status["id"], quote["id"] + ) + return + + in_reply: str | None = status.get("in_reply_to_id") + in_reply_to: str | None = status.get("in_reply_to_account_id") + if in_reply_to and in_reply_to != self.user_id: + return + + parent = None + if in_reply: + parent = self._get_post(self.url, self.user_id, in_reply) + if not parent: + self.log.info( + "Skipping %s, parent %s not found in db", status["id"], in_reply + ) + return + parser = StatusParser(status) + parser.feed(status["content"]) + tokens = parser.get_result() + + post = Post( + id=status["id"], + author=self.user_id, + service=self.url, + parent_id=in_reply, + tokens=tokens, + text_type="text/html", + ) + + if quote: + post.attachments.put( + QuoteAttachment(quoted_id=quote["id"], quoted_user=self.user_id) + ) + if status.get("url"): + post.attachments.put(RemoteUrlAttachment(url=status["url"])) + if status.get("sensitive"): + post.attachments.put(SensitiveAttachment(sensitive=True)) + if status.get("language"): + post.attachments.put(LanguagesAttachment(langs=[status["language"]])) + if status.get("spoiler_text"): + post.attachments.put(LabelsAttachment(labels=[status["spoiler_text"]])) + + blobs: list[Blob] = [] + for media in status.get("media_attachments", []): + self.log.info("Downloading %s...", media["url"]) + blob: Blob | None = download_blob( + media["url"], media.get("alt"), client=self.http + ) + if not blob: + self.log.error( + "Skipping %s! Failed to download media %s.", + status["id"], + media["url"], + ) + return + blobs.append(blob) + + if blobs: + post.attachments.put(MediaAttachment(blobs=blobs)) + + if parent: + self._insert_post( + { + "user": self.user_id, + "service": self.url, + "identifier": status["id"], + "parent": parent["id"], + "root": parent["id"] if not parent["root"] else parent["root"], + } + ) + else: + self._insert_post( + { + "user": self.user_id, + "service": self.url, + "identifier": status["id"], + } + ) + + self.log.info("Post stored in DB: %s", status["id"]) + + for out in self.outputs: + self.submitter(lambda: out.accept_post(post)) + + def _on_reblog(self, status: dict[str, Any], reblog: dict[str, Any]): + self.log.info("Processing reblog: %s", status["id"]) + reposted = self._get_post(self.url, self.user_id, reblog["id"]) + if not reposted: + self.log.info( + "Skipping repost '%s' as reposted post '%s' was not found in the db.", + status["id"], + reblog["id"], + ) + return + + self._insert_post( + { + "user": self.user_id, + "service": self.url, + "identifier": status["id"], + "reposted": reposted["id"], + } + ) + + self.log.info("Reblog stored in DB: %s", status["id"]) + + repost_ref = PostRef(id=status["id"], author=self.user_id, service=self.url) + reposted_ref = PostRef(id=reblog["id"], author=self.user_id, service=self.url) + for out in self.outputs: + self.submitter(lambda: out.accept_repost(repost_ref, reposted_ref)) + + def _on_delete_post(self, status_id: str): + self.log.info("Processing delete for %s...", status_id) + post = self._get_post(self.url, self.user_id, status_id) + if not post: + self.log.warning("Post not found in DB: %s", status_id) + return + + post_ref = PostRef(id=status_id, author=self.user_id, service=self.url) + if post["reposted"]: + self.log.info("Deleting repost: %s", status_id) + for output in self.outputs: + self.submitter(lambda: output.delete_repost(post_ref)) + else: + self.log.info("Deleting post: %s", status_id) + for output in self.outputs: + self.submitter(lambda: output.delete_post(post_ref)) + self.submitter(lambda: self._delete_post_by_id(post["id"])) + self.log.info("Delete processed successfully for %s", status_id) + + def _accept_msg(self, msg: websockets.Data) -> None: + data: dict[str, Any] = cast(dict[str, Any], json.loads(msg)) + event: str = cast(str, data["event"]) + payload: str = cast(str, data["payload"]) + + if event == "update": + self._on_create_post(json.loads(payload)) + elif event == "delete": + self._on_delete_post(payload) + + @override + async def listen(self): + url = f"{self.streaming_url}/api/v1/streaming?stream=user" + + async for ws in websockets.connect( + url, + additional_headers={"Authorization": f"Bearer {self.options.token}"}, + ping_interval=20, + ping_timeout=10, + close_timeout=5, + ): + try: + self.log.info("Listening to %s...", self.streaming_url) + + async def listen_for_messages(): + async for msg in ws: + self.submitter(lambda: self._accept_msg(msg)) + + listen = asyncio.create_task(listen_for_messages()) + + _ = await asyncio.gather(listen) + except websockets.ConnectionClosedError as e: + self.log.error(e, stack_info=True, exc_info=True) + self.log.info("Reconnecting to %s...", self.streaming_url) + continue + except TimeoutError as e: + self.log.error("Connection timeout: %s", e) + self.log.info("Reconnecting to %s...", self.streaming_url) + continue diff --git a/mastodon/output.py b/mastodon/output.py new file mode 100644 --- /dev/null +++ b/mastodon/output.py @@ -0,0 +1,567 @@ +import time +from dataclasses import dataclass +from typing import Any, override + +import httpx + +import misskey.mfm as mfm +from cross.attachments import ( + LanguagesAttachment, + MediaAttachment, + QuoteAttachment, + RemoteUrlAttachment, + SensitiveAttachment, +) +from cross.media import Blob +from cross.post import Post, PostRef +from cross.service import OutputService +from cross.tokens import LinkToken, TagToken, TextToken, Token +from database.connection import DatabasePool +from mastodon.info import InstanceInfo, MastodonService, validate_and_transform +from util.splitter import TokenSplitter, canonical_label + + +ALLOWED_POSTING_VISIBILITY: list[str] = ["public", "unlisted", "private"] +TEXT_MIMES: list[str] = ["text/x.misskeymarkdown", "text/markdown", "text/plain"] + + +@dataclass(kw_only=True) +class MastodonOutputOptions: + token: str + instance: str + visibility: str = "public" + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "MastodonOutputOptions": + validate_and_transform(data) + + if ( + "visibility" in data + and data["visibility"] not in ALLOWED_POSTING_VISIBILITY + ): + raise ValueError(f"Invalid visibility option {data['visibility']}!") + + return MastodonOutputOptions(**data) + + +@dataclass +class MediaUploadResult: + id: str + processed: bool = False + + +class MastodonOutputService(MastodonService, OutputService): + def __init__( + self, db: DatabasePool, http: httpx.Client, options: MastodonOutputOptions + ) -> None: + super().__init__(options.instance, db, http) + self.options: MastodonOutputOptions = options + + self.log.info("Verifying %s credentails...", self.url) + response = self.verify_credentials() + self.user_id: str = response["id"] + + self.log.info("Getting %s configuration...", self.url) + response = self.fetch_instance_info() + self.instance_info: InstanceInfo = InstanceInfo.from_api(response) + + def _token_to_string(self, tokens: list[Token]) -> str | None: + text: str = "" + for token in tokens: + match token: + case TextToken(): + text += token.text + case TagToken(): + text += f"#{token.tag}" + case LinkToken(): + if canonical_label(token.label, token.href): + text += token.href + else: + if self.instance_info.text_format == "text/plain": + if token.label: + text += f"{token.label} ({token.href})" + else: + text += token.href + elif self.instance_info.text_format in { + "text/x.misskeymarkdown", + "text/markdown", + }: + if token.label: + text += f"[{token.label}]({token.href})" + else: + text += token.href + else: + return None + return text + + def _split_tokens_and_media( + self, + tokens: list[Token], + media: list[Blob], + ) -> list[tuple[str, list[Blob]]] | None: + splitter = TokenSplitter( + max_chars=self.instance_info.max_characters, + max_link_len=self.instance_info.characters_reserved_per_url, + ) + split_token_blocks = splitter.split(tokens) + + if split_token_blocks is None: + return None + + post_texts: list[str] = [] + for block in split_token_blocks: + baked_text = self._token_to_string(block) + if baked_text is None: + return None + post_texts.append(baked_text) + + if not post_texts: + post_texts = [""] + + posts: list[dict[str, Any]] = [ + {"text": text, "attachments": []} for text in post_texts + ] + available_indices: list[int] = list(range(len(posts))) + current_image_post_idx: int | None = None + # video_post_idx: int | None = None + + def make_blank_post() -> dict[str, Any]: + return {"text": "", "attachments": []} + + def pop_next_empty_index() -> int: + if available_indices: + return available_indices.pop(0) + new_idx = len(posts) + posts.append(make_blank_post()) + return new_idx + + for blob in media: + if blob.mime.startswith(("video/", "audio/")): + current_image_post_idx = None + idx = pop_next_empty_index() + posts[idx]["attachments"].append(blob) + elif blob.mime.startswith("image/"): + if ( + current_image_post_idx is not None + and len(posts[current_image_post_idx]["attachments"]) + < self.instance_info.max_media_attachments + ): + posts[current_image_post_idx]["attachments"].append(blob) + else: + idx = pop_next_empty_index() + posts[idx]["attachments"].append(blob) + current_image_post_idx = idx + + result: list[tuple[str, list[Blob]]] = [] + for p in posts: + result.append((p["text"], p["attachments"])) + return result + + def _upload_media(self, attachments: list[Blob]) -> list[str] | None: + for blob in attachments: + if ( + blob.mime.startswith("image/") + and len(blob.io) > self.instance_info.image_size_limit + ): + self.log.error( + "Image too large: %s bytes (limit: %s)", + len(blob.io), + self.instance_info.image_size_limit, + ) + return None + if ( + blob.mime.startswith("video/") + and len(blob.io) > self.instance_info.video_size_limit + ): + self.log.error( + "Video too large: %s bytes (limit: %s)", + len(blob.io), + self.instance_info.video_size_limit, + ) + return None + if ( + not blob.mime.startswith(("image/", "video/")) + and len(blob.io) > 7_000_000 + ): + self.log.error("File too large: %s bytes", len(blob.io)) + return None + + uploads: list[MediaUploadResult] = [] + + for blob in attachments: + files = { + "file": ( + blob.name or "file", + blob.io, + blob.mime, + ) + } + data = {} + if blob.alt: + data["description"] = blob.alt + + response = self.http.post( + f"{self.url}/api/v2/media", + headers={"Authorization": f"Bearer {self._get_token()}"}, + files=files, + data=data, + ) + + if response.status_code == 200: + self.log.info( + "Uploaded %s! (%s)", blob.name or "unknown", response.json()["id"] + ) + uploads.append( + MediaUploadResult(id=response.json()["id"], processed=True) + ) + elif response.status_code == 202: + self.log.info("Waiting for %s to process!", blob.name or "unknown") + uploads.append( + MediaUploadResult(id=response.json()["id"], processed=False) + ) + else: + self.log.error( + "Failed to upload %s! %s", + blob.name or "unknown", + response.text, + ) + response.raise_for_status() + + while any(not result.processed for result in uploads): + self.log.info("Waiting for media to process...") + time.sleep(3) + for media_result in uploads: + if media_result.processed: + continue + response = self.http.get( + f"{self.url}/api/v1/media/{media_result.id}", + headers={"Authorization": f"Bearer {self._get_token()}"}, + ) + if response.status_code == 206: + continue + if response.status_code == 200: + media_result.processed = True + continue + response.raise_for_status() + + return [result.id for result in uploads] + + @override + def accept_post(self, post: Post): + self.log.info( + "Accepting post %s (author: %s, service: %s)...", + post.id, + post.author, + post.service, + ) + new_root_id: int | None = None + new_parent_id: int | None = None + + reply_ref: str | None = None + if post.parent_id: + thread = self._find_mapped_thread( + post.parent_id, post.service, post.author, self.url, self.user_id + ) + if not thread: + self.log.error("Failed to find thread tuple in the database!") + return + _, reply_ref, new_root_id, new_parent_id = thread + + quoted_status_id: str | None = None + quote = post.attachments.get(QuoteAttachment) + if quote: + if quote.quoted_user != post.author: + self.log.info("Quoted other user, skipping!") + return + + quoted_post = self._get_post(post.service, post.author, quote.quoted_id) + if not quoted_post: + self.log.error("Failed to find quoted post in the database!") + return + + quoted_mappings = self._get_mappings( + quoted_post["id"], self.url, self.user_id + ) + if not quoted_mappings: + self.log.error("Failed to find mappings for quoted post!") + return + + quoted_status_id = quoted_mappings[-1]["identifier"] + + post_tokens = post.tokens + if ( + post.text_type == "text/x.misskeymarkdown" + and self.instance_info.text_format != "text/x.misskeymarkdown" + ): + post_tokens, status = mfm.strip_mfm(post_tokens) + remote_url = post.attachments.get(RemoteUrlAttachment) + if status and remote_url and remote_url.url: + post_tokens.append(TextToken(text="\n")) + post_tokens.append( + LinkToken( + href=remote_url.url, label="[Post contains MFM, see original]" + ) + ) + + lang = "en" + langs = post.attachments.get(LanguagesAttachment) + if langs and langs.langs: + lang = langs.langs[0] + + sensitive = post.attachments.get(SensitiveAttachment) + + media_attachment = post.attachments.get(MediaAttachment) + media_blobs = media_attachment.blobs if media_attachment else [] + + raw_statuses = self._split_tokens_and_media(post_tokens, media_blobs) + if not raw_statuses: + self.log.error("Failed to split post into statuses!") + return + + baked_statuses: list[tuple[str, list[str] | None]] = [] + for status_text, raw_media in raw_statuses: + media_ids: list[str] | None = None + if raw_media: + media_ids = self._upload_media(raw_media) + if not media_ids: + self.log.error("Failed to upload attachments!") + return + baked_statuses.append((status_text, media_ids)) + + created_statuses: list[str] = [] + payload_sensitive = sensitive.sensitive if sensitive else False + + for i, (status_text, media_ids) in enumerate(baked_statuses): + payload: dict[str, Any] = { + "status": status_text or "", + "media_ids": media_ids or [], + "visibility": self.options.visibility, + "content_type": self.instance_info.text_format, + "language": lang, + } + + if media_ids or (sensitive and sensitive.sensitive): + payload["sensitive"] = payload_sensitive + + if sensitive and sensitive.sensitive: + payload["sensitive"] = True + + if reply_ref and i == 0: + payload["in_reply_to_id"] = reply_ref + + if quoted_status_id and i == 0: + payload["quoted_status_id"] = quoted_status_id + + response = self.http.post( + f"{self.url}/api/v1/statuses", + headers={ + "Authorization": f"Bearer {self._get_token()}", + "Content-Type": "application/json", + }, + json=payload, + ) + + if response.status_code != 200: + self.log.error( + "Failed to post status! %s - %s", + response.status_code, + response.text, + ) + response.raise_for_status() + + status_id = response.json()["id"] + self.log.info("Created new status %s!", status_id) + created_statuses.append(status_id) + + if i == 0: + reply_ref = status_id + + db_post = self._get_post(post.service, post.author, post.id) + if not db_post: + self.log.error("Post not found in database!") + return + + if new_root_id is None or new_parent_id is None: + self._insert_post( + { + "user": self.user_id, + "service": self.url, + "identifier": created_statuses[0], + "parent": None, + "root": None, + "reposted": None, + "extra_data": None, + "crossposted": 1, + } + ) + new_post = self._get_post(self.url, self.user_id, created_statuses[0]) + if not new_post: + raise ValueError("Inserted post not found!") + new_root_id = new_post["id"] + new_parent_id = new_root_id + + self._insert_post_mapping(db_post["id"], new_parent_id) + + for status_id in created_statuses[1:]: + self._insert_post( + { + "user": self.user_id, + "service": self.url, + "identifier": status_id, + "parent": new_parent_id, + "root": new_root_id, + "reposted": None, + "extra_data": None, + "crossposted": 1, + } + ) + reply_post = self._get_post(self.url, self.user_id, status_id) + if not reply_post: + raise ValueError("Inserted reply post not found!") + new_parent_id = reply_post["id"] + self._insert_post_mapping(db_post["id"], new_parent_id) + + self.log.info("Post accepted successfully: %s -> %s", post.id, created_statuses) + + @override + def delete_post(self, post: PostRef): + self.log.info( + "Deleting post %s (author: %s, service: %s)...", + post.id, + post.author, + post.service, + ) + db_post = self._get_post(post.service, post.author, post.id) + if not db_post: + self.log.warning( + "Post not found in DB: %s (author: %s, service: %s)", + post.id, + post.author, + post.service, + ) + return + + mappings = self._get_mappings(db_post["id"], self.url, self.user_id) + + for mapping in mappings[::-1]: + self.log.info("Deleting '%s'...", mapping["identifier"]) + self.http.delete( + f"{self.url}/api/v1/statuses/{mapping['identifier']}", + headers={"Authorization": f"Bearer {self._get_token()}"}, + ) + self._delete_post_by_id(mapping["id"]) + self.log.info("Post deleted successfully: %s", post.id) + + @override + def accept_repost(self, repost: PostRef, reposted: PostRef): + self.log.info( + "Accepting repost %s of %s (author: %s, service: %s)...", + repost.id, + reposted.id, + repost.author, + repost.service, + ) + original = self._get_post(reposted.service, reposted.author, reposted.id) + if not original: + self.log.info("Post not found in db, skipping repost..") + return + + mappings = self._get_mappings(original["id"], self.url, self.user_id) + if not mappings: + self.log.error("No mappings found for reposted post!") + return + + response = self.http.post( + f"{self.url}/api/v1/statuses/{mappings[0]['identifier']}/reblog", + headers={"Authorization": f"Bearer {self._get_token()}"}, + ) + + if response.status_code != 200: + self.log.error( + "Failed to boost status! status_code: %s, msg: %s", + response.status_code, + response.content, + ) + return + + self._insert_post( + { + "user": self.user_id, + "service": self.url, + "identifier": response.json()["id"], + "parent": None, + "root": None, + "reposted": mappings[0]["id"], + "extra_data": None, + "crossposted": 1, + } + ) + inserted = self._get_post(self.url, self.user_id, response.json()["id"]) + if not inserted: + raise ValueError("Inserted post not found!") + + original_repost = self._get_post(repost.service, repost.author, repost.id) + if not original_repost: + self.log.error("original repost not found in DB: %s", repost.id) + return + + self._insert_post_mapping(original_repost["id"], inserted["id"]) + self.log.info("Repost accepted successfully: %s", repost.id) + + @override + def delete_repost(self, repost: PostRef): + self.log.info( + "Deleting repost %s (author: %s, service: %s)...", + repost.id, + repost.author, + repost.service, + ) + db_repost = self._get_post(repost.service, repost.author, repost.id) + if not db_repost: + self.log.warning( + "Repost not found in DB: %s (author: %s, service: %s)", + repost.id, + repost.author, + repost.service, + ) + return + + mappings = self._get_mappings(db_repost["id"], self.url, self.user_id) + rmappings = self._get_mappings(db_repost["reposted"], self.url, self.user_id) + + if not mappings: + self.log.warning("No mappings found for repost %s", repost.id) + return + if not rmappings: + self.log.warning( + "No mappings found for original post %s (reposted_id=%s)", + repost.id, + db_repost["reposted"], + ) + return + + self.log.info( + "Removing '%s' Repost of '%s'...", + mappings[0]["identifier"], + rmappings[0]["identifier"], + ) + + response = self.http.post( + f"{self.url}/api/v1/statuses/{rmappings[0]['identifier']}/unreblog", + headers={"Authorization": f"Bearer {self._get_token()}"}, + ) + + if response.status_code != 200: + self.log.error( + "Failed to unreblog! status_code: %s, msg: %s", + response.status_code, + response.text, + ) + return + + self._delete_post_by_id(mappings[0]["id"]) + self.log.info("Repost deleted successfully: %s", repost.id) + + @override + def _get_token(self) -> str: + return self.options.token diff --git a/migrations/001_initdb_v1.py b/migrations/001_initdb_v1.py new file mode 100644 --- /dev/null +++ b/migrations/001_initdb_v1.py @@ -0,0 +1,21 @@ +import sqlite3 + + +def migrate(conn: sqlite3.Connection): + _ = conn.execute(""" + CREATE TABLE IF NOT EXISTS posts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT NOT NULL, + service TEXT NOT NULL, + identifier TEXT NOT NULL, + parent_id INTEGER NULL REFERENCES posts(id) ON DELETE SET NULL, + root_id INTEGER NULL REFERENCES posts(id) ON DELETE SET NULL + ); + """) + _ = conn.execute(""" + CREATE TABLE IF NOT EXISTS mappings ( + original_post_id INTEGER NOT NULL REFERENCES posts(id) ON DELETE CASCADE, + mapped_post_id INTEGER NOT NULL + ); + """) + pass diff --git a/migrations/002_add_reposted_column_v1.py b/migrations/002_add_reposted_column_v1.py new file mode 100644 --- /dev/null +++ b/migrations/002_add_reposted_column_v1.py @@ -0,0 +1,11 @@ +import sqlite3 + + +def migrate(conn: sqlite3.Connection): + columns = conn.execute("PRAGMA table_info(posts)") + column_names = [col[1] for col in columns] + if "reposted_id" not in column_names: + _ = conn.execute(""" + ALTER TABLE posts + ADD COLUMN reposted_id INTEGER NULL REFERENCES posts(id) ON DELETE SET NULL + """) diff --git a/migrations/003_add_extra_data_column_v1.py b/migrations/003_add_extra_data_column_v1.py new file mode 100644 --- /dev/null +++ b/migrations/003_add_extra_data_column_v1.py @@ -0,0 +1,26 @@ +import json +import sqlite3 + + +def migrate(conn: sqlite3.Connection): + columns = conn.execute("PRAGMA table_info(posts)") + column_names = [col[1] for col in columns] + if "extra_data" not in column_names: + _ = conn.execute(""" + ALTER TABLE posts + ADD COLUMN extra_data TEXT NULL + """) + + # migrate old bsky identifiers from json to uri as id and cid in extra_data + data = conn.execute( + "SELECT id, identifier FROM posts WHERE service = 'https://bsky.app';" + ).fetchall() + rewrites: list[tuple[str, str, int]] = [] + for row in data: + if row[1][0] == "{" and row[1][-1] == "}": + data = json.loads(row[1]) + rewrites.append((data["uri"], json.dumps({"cid": data["cid"]}), row[0])) + if rewrites: + _ = conn.executemany( + "UPDATE posts SET identifier = ?, extra_data = ? WHERE id = ?;", rewrites + ) diff --git a/migrations/004_initdb_next.py b/migrations/004_initdb_next.py new file mode 100644 --- /dev/null +++ b/migrations/004_initdb_next.py @@ -0,0 +1,52 @@ +import sqlite3 + + +def migrate(conn: sqlite3.Connection): + cursor = conn.cursor() + + old_posts = cursor.execute("SELECT * FROM posts;").fetchall() + old_mappings = cursor.execute("SELECT * FROM mappings;").fetchall() + + _ = cursor.execute("DROP TABLE posts;") + _ = cursor.execute("DROP TABLE mappings;") + + _ = cursor.execute(""" + CREATE TABLE posts ( + id INTEGER UNIQUE PRIMARY KEY AUTOINCREMENT, + user TEXT NOT NULL, + service TEXT NOT NULL, + identifier TEXT NOT NULL, + parent INTEGER NULL REFERENCES posts(id), + root INTEGER NULL REFERENCES posts(id), + reposted INTEGER NULL REFERENCES posts(id), + extra_data TEXT NULL + ); + """) + + _ = cursor.execute(""" + CREATE TABLE mappings ( + original INTEGER NOT NULL REFERENCES posts(id) ON DELETE CASCADE, + mapped INTEGER NOT NULL REFERENCES posts(id) ON DELETE CASCADE, + UNIQUE(original, mapped) + ); + """) + + for old_post in old_posts: + _ = cursor.execute( + """ + INSERT INTO posts (id, user, service, identifier, parent, root, reposted, extra_data) + VALUES (:id, :user_id, :service, :identifier, :parent_id, :root_id, :reposted_id, :extra_data) + """, + dict(old_post), + ) + + for mapping in old_mappings: + original, mapped = mapping["original_post_id"], mapping["mapped_post_id"] + _ = cursor.execute( + "INSERT OR IGNORE INTO mappings (original, mapped) VALUES (?, ?)", + (original, mapped), + ) + _ = cursor.execute( + "INSERT OR IGNORE INTO mappings (original, mapped) VALUES (?, ?)", + (mapped, original), + ) diff --git a/migrations/005_add_indexes.py b/migrations/005_add_indexes.py new file mode 100644 --- /dev/null +++ b/migrations/005_add_indexes.py @@ -0,0 +1,12 @@ +import sqlite3 + + +def migrate(conn: sqlite3.Connection): + _ = conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_posts_service_user_identifier + ON posts (service, user, identifier); + """) + _ = conn.execute(""" + CREATE UNIQUE INDEX IF NOT EXISTS ux_mappings_original_mapped + ON mappings (original, mapped); + """) diff --git a/migrations/006_add_atproto_tables.py b/migrations/006_add_atproto_tables.py new file mode 100644 --- /dev/null +++ b/migrations/006_add_atproto_tables.py @@ -0,0 +1,35 @@ +import sqlite3 + + +def migrate(conn: sqlite3.Connection): + _ = conn.execute(""" + CREATE TABLE IF NOT EXISTS atproto_sessions ( + did TEXT PRIMARY KEY, + pds TEXT NOT NULL, + handle TEXT NOT NULL, + access_jwt TEXT NOT NULL, + refresh_jwt TEXT NOT NULL, + email TEXT, + email_confirmed INTEGER DEFAULT 0, + email_auth_factor INTEGER DEFAULT 0, + active INTEGER DEFAULT 1, + status TEXT, + created_at REAL NOT NULL + ) + """) + _ = conn.execute(""" + CREATE TABLE IF NOT EXISTS atproto_identities ( + identifier TEXT PRIMARY KEY, + did TEXT NOT NULL, + handle TEXT NOT NULL, + pds TEXT NOT NULL, + signing_key TEXT NOT NULL, + created_at REAL NOT NULL + ) + """) + _ = conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_sessions_pds ON atproto_sessions(pds) + """) + _ = conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_identities_created_at ON atproto_identities(created_at) + """) diff --git a/migrations/007_add_crossposted_column.py b/migrations/007_add_crossposted_column.py new file mode 100644 --- /dev/null +++ b/migrations/007_add_crossposted_column.py @@ -0,0 +1,10 @@ +import sqlite3 + + +def migrate(conn: sqlite3.Connection): + _ = conn.execute(""" + ALTER TABLE posts ADD COLUMN crossposted INTEGER DEFAULT 0 + """) + _ = conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_posts_crossposted ON posts(crossposted) + """) diff --git a/migrations/__init__.py b/migrations/__init__.py new file mode 100644 --- /dev/null +++ b/migrations/__init__.py diff --git a/migrations/_registry.py b/migrations/_registry.py new file mode 100644 --- /dev/null +++ b/migrations/_registry.py @@ -0,0 +1,37 @@ +import importlib.util +import sqlite3 +from collections.abc import Callable +from pathlib import Path + + +def load_migrations( + path: Path, +) -> list[tuple[int, str, Callable[[sqlite3.Connection], None]]]: + migrations: list[tuple[int, str, Callable[[sqlite3.Connection], None]]] = [] + migration_files = sorted( + [f for f in path.glob("*.py") if not f.stem.startswith("_")] + ) + + for filepath in migration_files: + filename = filepath.stem + version_str = filename.split("_")[0] + + try: + version = int(version_str) + except ValueError: + raise ValueError("migrations must start with a number!!") + + spec = importlib.util.spec_from_file_location(filepath.stem, filepath) + if not spec or not spec.loader: + raise Exception(f"Failed to load spec from file: {filepath}") + + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + if hasattr(module, "migrate"): + migrations.append((version, filename, module.migrate)) + else: + raise ValueError(f"Migration {filepath.name} missing 'migrate' function") + + migrations.sort(key=lambda x: x[0]) + return migrations diff --git a/misskey/__init__.py b/misskey/__init__.py new file mode 100644 --- /dev/null +++ b/misskey/__init__.py diff --git a/misskey/info.py b/misskey/info.py new file mode 100644 --- /dev/null +++ b/misskey/info.py @@ -0,0 +1,27 @@ +from abc import ABC, abstractmethod + +import httpx + +from cross.service import Service +from database.connection import DatabasePool + + +class MisskeyService(ABC, Service): + def __init__(self, url: str, db: DatabasePool, http: httpx.Client) -> None: + super().__init__(url, db) + self.http = http + + def verify_credentials(self): + response = self.http.post( + f"{self.url}/api/i", + json={"i": self._get_token()}, + headers={"Content-Type": "application/json"}, + ) + if response.status_code != 200: + self.log.error("Failed to validate user credentials!") + response.raise_for_status() + return dict(response.json()) + + @abstractmethod + def _get_token(self) -> str: + pass diff --git a/misskey/input.py b/misskey/input.py new file mode 100644 --- /dev/null +++ b/misskey/input.py @@ -0,0 +1,270 @@ +import asyncio +import json +import re +import uuid +from dataclasses import dataclass, field +from typing import Any, cast, override + +import httpx +import websockets + +from cross.attachments import ( + LabelsAttachment, + MediaAttachment, + QuoteAttachment, + RemoteUrlAttachment, + SensitiveAttachment, +) +from cross.media import Blob, download_blob +from cross.post import Post, PostRef +from cross.service import InputService +from database.connection import DatabasePool +from misskey.info import MisskeyService +from util.markdown import MarkdownParser +from util.util import normalize_service_url + + +ALLOWED_VISIBILITY = ["public", "home"] + + +@dataclass +class MisskeyInputOptions: + token: str + instance: str + allowed_visibility: list[str] = field( + default_factory=lambda: ALLOWED_VISIBILITY.copy() + ) + filters: list[re.Pattern[str]] = field(default_factory=lambda: []) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "MisskeyInputOptions": + data["instance"] = normalize_service_url(data["instance"]) + + if "allowed_visibility" in data: + for vis in data.get("allowed_visibility", []): + if vis not in ALLOWED_VISIBILITY: + raise ValueError(f"Invalid visibility option {vis}!") + + if "filters" in data: + data["filters"] = [re.compile(r) for r in data["filters"]] + + return MisskeyInputOptions(**data) + + +class MisskeyInputService(MisskeyService, InputService): + def __init__( + self, db: DatabasePool, http: httpx.Client, options: MisskeyInputOptions + ) -> None: + super().__init__(options.instance, db, http) + self.options: MisskeyInputOptions = options + + self.log.info("Verifying %s credentails...", self.url) + response = self.verify_credentials() + self.user_id: str = response["id"] + + @override + def _get_token(self) -> str: + return self.options.token + + def _on_note(self, note: dict[str, Any]): + self.log.info("Processing new note: %s", note["id"]) + + if note["userId"] != self.user_id: + return + + if note["visibility"] not in self.options.allowed_visibility: + self.log.info( + "Skipping note with disallowed visibility: %s (%s)", + note["id"], + note["visibility"], + ) + return + + if self._is_post_crossposted(self.url, self.user_id, note["id"]): + self.log.info( + "Skipping %s, already crossposted", + note["id"], + ) + return + + if note.get("poll"): + self.log.info("Skipping '%s'! Contains a poll..", note["id"]) + return + + renote: dict[str, Any] | None = note.get("renote") + if renote: + if note.get("text") is None: + self._on_renote(note, renote) + return + + if renote["userId"] != self.user_id: + return + + rrenote = self._get_post(self.url, self.user_id, renote["id"]) + if not rrenote: + self.log.info( + "Skipping %s, quote %s not found in db", note["id"], renote["id"] + ) + return + + reply: dict[str, Any] | None = note.get("reply") + if reply and reply.get("userId") != self.user_id: + self.log.info("Skipping '%s'! Reply to other user..", note["id"]) + return + + parent = None + if reply: + parent = self._get_post(self.url, self.user_id, reply["id"]) + if not parent: + self.log.info( + "Skipping %s, parent %s not found in db", note["id"], reply["id"] + ) + return + + mention_handles: dict = note.get("mentionHandles") or {} + tags: list[str] = note.get("tags") or [] + + handles: list[tuple[str, str]] = [] + for _key, value in mention_handles.items(): + handles.append((value, value)) + + parser = MarkdownParser() # TODO MFM parser + tokens = parser.parse(note.get("text", ""), tags, handles) + post = Post( + id=note["id"], + author=self.user_id, + service=self.url, + parent_id=reply["id"] if reply else None, + tokens=tokens, + text_type="text/x.misskeymarkdown", + ) + + post.attachments.put(RemoteUrlAttachment(url=self.url + "/notes/" + note["id"])) + if renote: + post.attachments.put( + QuoteAttachment(quoted_id=renote["id"], quoted_user=self.user_id) + ) + if any(a.get("isSensitive", False) for a in note.get("files", [])): + post.attachments.put(SensitiveAttachment(sensitive=True)) + if note.get("cw"): + post.attachments.put(LabelsAttachment(labels=[note["cw"]])) + + blobs: list[Blob] = [] + for media in note.get("files", []): + self.log.info("Downloading %s...", media["url"]) + blob: Blob | None = download_blob( + media["url"], media.get("comment", ""), client=self.http + ) + if not blob: + self.log.error( + "Skipping %s! Failed to download media %s.", + note["id"], + media["url"], + ) + return + blobs.append(blob) + + if blobs: + post.attachments.put(MediaAttachment(blobs=blobs)) + + if parent: + self._insert_post( + { + "user": self.user_id, + "service": self.url, + "identifier": note["id"], + "parent": parent["id"], + "root": parent["id"] if not parent["root"] else parent["root"], + } + ) + else: + self._insert_post( + { + "user": self.user_id, + "service": self.url, + "identifier": note["id"], + } + ) + + self.log.info("Note stored in DB: %s", note["id"]) + + for out in self.outputs: + self.submitter(lambda: out.accept_post(post)) + + def _on_renote(self, note: dict[str, Any], renote: dict[str, Any]): + self.log.info("Processing renote: %s", note["id"]) + reposted = self._get_post(self.url, self.user_id, renote["id"]) + if not reposted: + self.log.info( + "Skipping repost '%s' as reposted post '%s' was not found in the db.", + note["id"], + renote["id"], + ) + return + + self._insert_post( + { + "user": self.user_id, + "service": self.url, + "identifier": note["id"], + "reposted": reposted["id"], + } + ) + + self.log.info("Renote stored in DB: %s", note["id"]) + + repost_ref = PostRef(id=note["id"], author=self.user_id, service=self.url) + reposted_ref = PostRef(id=renote["id"], author=self.user_id, service=self.url) + for out in self.outputs: + self.submitter(lambda: out.accept_repost(repost_ref, reposted_ref)) + + def _accept_msg(self, msg: websockets.Data) -> None: + data: dict[str, Any] = cast(dict[str, Any], json.loads(msg)) + + if data["type"] == "channel": + type: str = cast(str, data["body"]["type"]) + if type == "note" or type == "reply": + note_body = data["body"]["body"] + self._on_note(note_body) + + async def _subscribe_to_home(self, ws: websockets.ClientConnection) -> None: + await ws.send( + json.dumps( + { + "type": "connect", + "body": {"channel": "homeTimeline", "id": str(uuid.uuid4())}, + } + ) + ) + self.log.info("Subscribed to 'homeTimeline' channel...") + + @override + async def listen(self): + streaming: str = f"{'wss' if self.url.startswith('https') else 'ws'}://{self.url.split('://', 1)[1]}" + url: str = f"{streaming}/streaming?i={self.options.token}" + + async for ws in websockets.connect( + url, + ping_interval=20, + ping_timeout=10, + close_timeout=5, + ): + try: + self.log.info("Listening to %s...", streaming) + await self._subscribe_to_home(ws) + + async def listen_for_messages(): + async for msg in ws: + self.submitter(lambda: self._accept_msg(msg)) + + listen = asyncio.create_task(listen_for_messages()) + + _ = await asyncio.gather(listen) + except websockets.ConnectionClosedError as e: + self.log.error(e, stack_info=True, exc_info=True) + self.log.info("Reconnecting to %s...", streaming) + continue + except TimeoutError as e: + self.log.error("Connection timeout: %s", e) + self.log.info("Reconnecting to %s...", streaming) + continue diff --git a/misskey/mfm.py b/misskey/mfm.py new file mode 100644 --- /dev/null +++ b/misskey/mfm.py @@ -0,0 +1,43 @@ +import re + +from cross.tokens import LinkToken, TextToken, Token + + +MFM_PATTERN = re.compile(r"\$\[([^\[\]]+)\]") + + +def strip_mfm(tokens: list[Token]) -> tuple[list[Token], bool]: + modified = False + original: str | None + + for tk in tokens: + if isinstance(tk, TextToken): + original = tk.text + cleaned = __strip_mfm(original) + if cleaned != original: + modified = True + tk.text = cleaned or "" + + elif isinstance(tk, LinkToken): + original = tk.label + cleaned = __strip_mfm(original) + if cleaned != original: + modified = True + tk.label = cleaned + + return tokens, modified + + +def __strip_mfm(text: str | None) -> str | None: + if text is None: + return None + + def match_contents(match: re.Match[str]): + content = match.group(1).strip() + parts = content.split(" ", 1) + return parts[1] if len(parts) > 1 else "" + + while MFM_PATTERN.search(text): + text = MFM_PATTERN.sub(match_contents, text) + + return text diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,74 @@ +[project] +name = "xpost" +version = "0.1.0" +description = "social media crossposting tool" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "dnspython>=2.8.0", + "grapheme>=0.6.0", + "httpx>=0.27.0", + "python-magic>=0.4.27", + "websockets>=15.0.1", +] + +[dependency-groups] +dev = [ + "pytest>=8.4.2", + "ruff>=0.9.0", + "mypy>=1.15.0", +] + +[tool.pytest.ini_options] +pythonpath = ["."] + +[tool.ruff] +target-version = "py312" +line-length = 88 +src = ["."] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # Pyflakes + "I", # isort + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "UP", # pyupgrade + "ARG", # flake8-unused-arguments + "SIM", # flake8-simplify +] +ignore = [ + "E501", # line too long (handled by formatter) + "B008", # do not perform function calls in argument defaults + "B904", # raise without from inside except + "B023", # loop variable binding in lambdas (false positive for async for) +] + +[tool.ruff.lint.isort] +force-single-line = false +lines-after-imports = 2 + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" + +[tool.mypy] +python_version = "3.12" +warn_return_any = true +warn_unused_configs = true +warn_unused_ignores = true +disallow_untyped_defs = false +disallow_incomplete_defs = false +check_untyped_defs = true +no_implicit_optional = true +warn_redundant_casts = true +strict_equality = true + +# external libraries +[[tool.mypy.overrides]] +module = ["grapheme"] +ignore_missing_imports = true diff --git a/registry.py b/registry.py new file mode 100644 --- /dev/null +++ b/registry.py @@ -0,0 +1,43 @@ +from collections.abc import Callable +from typing import Any + +import httpx + +from cross.service import InputService, OutputService +from database.connection import DatabasePool + + +input_factories: dict[ + str, Callable[[DatabasePool, httpx.Client, dict[str, Any]], InputService] +] = {} +output_factories: dict[ + str, Callable[[DatabasePool, httpx.Client, dict[str, Any]], OutputService] +] = {} + + +def create_input_service( + db: DatabasePool, http: httpx.Client, data: dict[str, Any] +) -> InputService: + if "type" not in data: + raise ValueError("No `type` field in input data!") + type: str = str(data["type"]) + del data["type"] + + factory = input_factories.get(type) + if not factory: + raise KeyError(f"No such input service {type}!") + return factory(db, http, data) + + +def create_output_service( + db: DatabasePool, http: httpx.Client, data: dict[str, Any] +) -> OutputService: + if "type" not in data: + raise ValueError("No `type` field in input data!") + type: str = str(data["type"]) + del data["type"] + + factory = output_factories.get(type) + if not factory: + raise KeyError(f"No such output service {type}!") + return factory(db, http, data) diff --git a/registry_bootstrap.py b/registry_bootstrap.py new file mode 100644 --- /dev/null +++ b/registry_bootstrap.py @@ -0,0 +1,42 @@ +from typing import Any + +import httpx + +from database.connection import DatabasePool +from registry import input_factories, output_factories + + +class LazyFactory: + def __init__(self, module_path: str, class_name: str, options_class_name: str): + self.module_path: str = module_path + self.class_name: str = class_name + self.options_class_name: str = options_class_name + + def __call__(self, db: DatabasePool, http: httpx.Client, d: dict[str, Any]): + module = __import__( + self.module_path, fromlist=[self.class_name, self.options_class_name] + ) + service_class = getattr(module, self.class_name) + options_class = getattr(module, self.options_class_name) + return service_class(db, http, options_class.from_dict(d)) + + +def bootstrap(): + input_factories["mastodon-wss"] = LazyFactory( + "mastodon.input", "MastodonInputService", "MastodonInputOptions" + ) + input_factories["misskey-wss"] = LazyFactory( + "misskey.input", "MisskeyInputService", "MisskeyInputOptions" + ) + input_factories["bluesky-jetstream"] = LazyFactory( + "bluesky.input", "BlueskyJetstreamInputService", "BlueskyInputOptions" + ) + output_factories["stderr"] = LazyFactory( + "util.dummy", "StderrOutputService", "DummyOptions" + ) + output_factories["bluesky"] = LazyFactory( + "bluesky.output", "BlueskyOutputService", "BlueskyOutputOptions" + ) + output_factories["mastodon"] = LazyFactory( + "mastodon.output", "MastodonOutputService", "MastodonOutputOptions" + ) diff --git a/tests/util/util_test.py b/tests/util/util_test.py new file mode 100644 --- /dev/null +++ b/tests/util/util_test.py @@ -0,0 +1,63 @@ +from unittest.mock import patch + +import pytest + +import util.util as u + + +def test_normalize_service_url_http(): + assert u.normalize_service_url("http://example.com") == "http://example.com" + assert u.normalize_service_url("http://example.com/") == "http://example.com" + + +def test_normalize_service_url_invalid_schemes(): + with pytest.raises(ValueError, match="Invalid service url"): + _ = u.normalize_service_url("ftp://example.com") + with pytest.raises(ValueError, match="Invalid service url"): + _ = u.normalize_service_url("example.com") + with pytest.raises(ValueError, match="Invalid service url"): + _ = u.normalize_service_url("//example.com") + + +def test_read_env_missing_env_var(): + data = {"token": "env:MISSING_VAR", "keep": "value"} + with patch.dict("os.environ", {}, clear=True): + u.read_env(data) + assert data == {"keep": "value"} + assert "token" not in data + + +def test_read_env_no_env_prefix(): + data = {"token": "literal_value", "number": 123} + u.read_env(data) + assert data == {"token": "literal_value", "number": 123} + + +def test_read_env_deeply_nested(): + data = {"level1": {"level2": {"token": "env:DEEP_TOKEN"}}} + with patch.dict("os.environ", {"DEEP_TOKEN": "deep_secret"}): + u.read_env(data) + assert data["level1"]["level2"]["token"] == "deep_secret" + + +def test_read_env_mixed_types(): + data: dict[str, object] = { + "string": "env:TOKEN", + "number": 42, + "list": [1, 2, 3], + "none": None, + "bool": True, + } + with patch.dict("os.environ", {"TOKEN": "secret"}): + u.read_env(data) + assert data["string"] == "secret" + assert data["number"] == 42 + assert data["list"] == [1, 2, 3] + assert data["none"] is None + assert data["bool"] is True + + +def test_read_env_empty_dict(): + data: dict[str, object] = {} + u.read_env(data) + assert data == {} diff --git a/util/__init__.py b/util/__init__.py new file mode 100644 --- /dev/null +++ b/util/__init__.py diff --git a/util/cache.py b/util/cache.py new file mode 100644 --- /dev/null +++ b/util/cache.py @@ -0,0 +1,48 @@ +import pickle +import time +from abc import ABC, abstractmethod +from pathlib import Path +from typing import override + + +class Cacheable(ABC): + @abstractmethod + def dump_cache(self, path: Path): + pass + + @abstractmethod + def load_cache(self, path: Path): + pass + + +class TTLCache[K, V](Cacheable): + def __init__(self, ttl_seconds: int = 3600) -> None: + self.ttl: int = ttl_seconds + self.__cache: dict[K, tuple[V, float]] = {} + + def get(self, key: K) -> V | None: + if key in self.__cache: + value, timestamp = self.__cache[key] + if time.time() - timestamp < self.ttl: + return value + else: + del self.__cache[key] + return None + + def set(self, key: K, value: V) -> None: + self.__cache[key] = (value, time.time()) + + def clear(self) -> None: + self.__cache.clear() + + @override + def dump_cache(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "wb") as f: + pickle.dump(self.__cache, f) + + @override + def load_cache(self, path: Path): + if path.exists(): + with open(path, "rb") as f: + self.__cache = pickle.load(f) diff --git a/util/dummy.py b/util/dummy.py new file mode 100644 --- /dev/null +++ b/util/dummy.py @@ -0,0 +1,32 @@ +from typing import override + +from cross.post import Post, PostRef +from cross.service import OutputService +from database.connection import DatabasePool + + +class DummyOptions: + @classmethod + def from_dict(cls, _obj) -> "DummyOptions": + return DummyOptions() + + +class StderrOutputService(OutputService): + def __init__(self, db: DatabasePool, _options: DummyOptions) -> None: + super().__init__("http://localhost", db) + + @override + def accept_post(self, post: Post): + self.log.info("%s", post) + + @override + def accept_repost(self, repost: PostRef, reposted: PostRef): + self.log.info("%s, %s", repost.id, reposted.id) + + @override + def delete_post(self, post: PostRef): + self.log.info("%s", post.id) + + @override + def delete_repost(self, repost: PostRef): + self.log.info("%s", repost.id) diff --git a/util/html.py b/util/html.py new file mode 100644 --- /dev/null +++ b/util/html.py @@ -0,0 +1,151 @@ +from html.parser import HTMLParser +from typing import override + +from cross.tokens import LinkToken, TextToken, Token +from util.splitter import canonical_label + + +class HTMLToTokensParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.tokens: list[Token] = [] + + self._tag_stack: dict[str, tuple[str, dict[str, str | None]]] = {} + self.in_pre: bool = False + self.in_code: bool = False + self.invisible: bool = False + + def handle_a_endtag(self): + label, _attr = self._tag_stack.pop("a") + + href = _attr.get("href") + if href: + if canonical_label(label, href): + self.tokens.append(LinkToken(href=href)) + else: + self.tokens.append(LinkToken(href=href, label=label)) + + def append_text(self, text: str): + self.tokens.append(TextToken(text=text)) + + def append_newline(self): + if self.tokens: + last_token = self.tokens[-1] + if isinstance(last_token, TextToken) and not last_token.text.endswith("\n"): + self.tokens.append(TextToken(text="\n")) + + @override + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + _attr = dict(attrs) + + if self.invisible: + return + + match tag: + case "p": + cls = _attr.get("class", "") + if cls and "quote-inline" in cls: + self.invisible = True + case "a": + self._tag_stack["a"] = ("", _attr) + case "code": + if not self.in_pre: + self.append_text("`") + self.in_code = True + case "pre": + self.append_newline() + self.append_text("```\n") + self.in_pre = True + case "blockquote": + self.append_newline() + self.append_text("> ") + case "strong" | "b": + self.append_text("**") + case "em" | "i": + self.append_text("*") + case "del" | "s": + self.append_text("~~") + case "br": + self.append_text("\n") + case "h1" | "h2" | "h3" | "h4" | "h5" | "h6": + level = int(tag[1]) + self.append_text("\n" + "#" * level + " ") + case _: + # self.builder.extend(f"<{tag}>".encode("utf-8")) + pass + + @override + def handle_endtag(self, tag: str) -> None: + if self.invisible: + if tag == "p": + self.invisible = False + return + + match tag: + case "a": + if "a" in self._tag_stack: + self.handle_a_endtag() + case "code": + if not self.in_pre and self.in_code: + self.append_text("`") + self.in_code = False + case "pre": + self.append_newline() + self.append_text("```\n") + self.in_pre = False + case "blockquote": + self.append_text("\n") + case "strong" | "b": + self.append_text("**") + case "em" | "i": + self.append_text("*") + case "del" | "s": + self.append_text("~~") + case "p": + self.append_text("\n\n") + case "h1" | "h2" | "h3" | "h4" | "h5" | "h6": + self.append_text("\n") + case _: + # self.builder.extend(f"".encode("utf-8")) + pass + + @override + def handle_data(self, data: str) -> None: + if self.invisible: + return + + if self._tag_stack.get("a"): + label, _attr = self._tag_stack.pop("a") + self._tag_stack["a"] = (label + data, _attr) + else: + self.append_text(data) + + def get_result(self) -> list[Token]: + if not self.tokens: + return [] + + combined: list[Token] = [] + buffer: list[str] = [] + + def flush_buffer(): + if buffer: + merged = "".join(buffer) + combined.append(TextToken(text=merged)) + buffer.clear() + + for token in self.tokens: + if isinstance(token, TextToken): + buffer.append(token.text) + else: + flush_buffer() + combined.append(token) + + flush_buffer() + + if combined and isinstance(combined[-1], TextToken): + if combined[-1].text.endswith("\n\n"): + combined[-1] = TextToken(text=combined[-1].text[:-2]) + + if combined[-1].text.endswith("\n"): + combined[-1] = TextToken(text=combined[-1].text[:-1]) + return combined diff --git a/util/markdown.py b/util/markdown.py new file mode 100644 --- /dev/null +++ b/util/markdown.py @@ -0,0 +1,127 @@ +import re + +from cross.tokens import LinkToken, MentionToken, TagToken, TextToken, Token +from util.html import HTMLToTokensParser +from util.splitter import canonical_label + + +URL = re.compile(r"(?:(?:[A-Za-z][A-Za-z0-9+.-]*://)|mailto:)[^\s]+", re.IGNORECASE) +MD_INLINE_LINK = re.compile( + r"\[([^\]]+)\]\(\s*((?:(?:[A-Za-z][A-Za-z0-9+.\-]*://)|mailto:)[^\s\)]+)\s*\)", + re.IGNORECASE, +) +MD_AUTOLINK = re.compile( + r"<((?:(?:[A-Za-z][A-Za-z0-9+.\-]*://)|mailto:)[^\s>]+)>", re.IGNORECASE +) +HASHTAG = re.compile(r"(? list[Token]: + if not text: + return [] + + tokenizer = HTMLToTokensParser() + tokenizer.feed(text) + html_tokens = tokenizer.get_result() + + tokens: list[Token] = [] + + for tk in html_tokens: + if isinstance(tk, TextToken): + tokens.extend(self.__tokenize_md(tk.text, tags, handles)) + elif isinstance(tk, LinkToken): + if not tk.label or canonical_label(tk.label, tk.href): + tokens.append(tk) + continue + + tokens.extend( + self.__tokenize_md(f"[{tk.label}]({tk.href})", tags, handles) + ) + else: + tokens.append(tk) + + return tokens + + def __tokenize_md( + self, text: str, tags: list[str], handles: list[tuple[str, str]] + ) -> list[Token]: + index: int = 0 + total: int = len(text) + buffer: list[str] = [] + + tokens: list[Token] = [] + + def flush(): + nonlocal buffer + if buffer: + tokens.append(TextToken(text="".join(buffer))) + buffer = [] + + while index < total: + if text[index] == "[": + md_inline = MD_INLINE_LINK.match(text, index) + if md_inline: + flush() + label = md_inline.group(1) + href = md_inline.group(2) + tokens.append(LinkToken(href=href, label=label)) + index = md_inline.end() + continue + + if text[index] == "<": + md_auto = MD_AUTOLINK.match(text, index) + if md_auto: + flush() + href = md_auto.group(1) + tokens.append(LinkToken(href=href, label=None)) + index = md_auto.end() + continue + + if text[index] == "#": + tag = HASHTAG.match(text, index) + if tag: + tag_text = tag.group(1) + if tag_text.lower() in tags: + flush() + tokens.append(TagToken(tag=tag_text)) + index = tag.end() + continue + + if text[index] == "@": + handle = FEDIVERSE_HANDLE.match(text, index) + if handle: + handle_text = handle.group(0) + stripped_handle = handle_text.strip() + + match = next( + (pair for pair in handles if stripped_handle in pair), None + ) + + if match: + flush() + tokens.append( + MentionToken(username=match[1], uri=None) + ) # TODO: misskey doesn’t provide a uri + index = handle.end() + continue + + url = URL.match(text, index) + if url: + flush() + href = url.group(0) + tokens.append(LinkToken(href=href, label=None)) + index = url.end() + continue + + buffer.append(text[index]) + index += 1 + + flush() + return tokens diff --git a/util/splitter.py b/util/splitter.py new file mode 100644 --- /dev/null +++ b/util/splitter.py @@ -0,0 +1,187 @@ +import re +from functools import lru_cache + +import grapheme + +from cross.tokens import LinkToken, TagToken, TextToken, Token + + +def canonical_label(label: str | None, href: str): + if not label or label == href: + return True + split = href.split("://", 1) + return len(split) > 1 and split[1] == label + + +@lru_cache(maxsize=1024) +def _grapheme_length(text: str) -> int: + return int(grapheme.length(text)) + + +TEXT_SPLITTER = re.compile(r"(\n{2,}|[!.,;:]+|\s+|[^\s!.,;:\n]+)") + + +class TokenSplitter: + def __init__(self, max_chars: int, max_link_len: int = 35): + self.max_chars = max_chars + self.max_link_len = max_link_len + self.blocks: list[list[Token]] = [] + self.current_block: list[Token] = [] + self.current_length = 0 + self.best_split_idx: tuple[int, int, int] | None = None + + def _save_block(self): + if self.current_block: + self.blocks.append(self.current_block) + self.current_block = [] + self.current_length = 0 + self.best_split_idx = None + + def _get_token_length(self, token: Token) -> int: + if isinstance(token, TextToken): + return _grapheme_length(token.text) + elif isinstance(token, LinkToken): + if token.label: + return ( + min(_grapheme_length(token.label), self.max_link_len) + if canonical_label(token.label, token.href) + else _grapheme_length(token.label) + ) + return min(_grapheme_length(token.href), self.max_link_len) + elif isinstance(token, TagToken): + return 1 + _grapheme_length(token.tag) + return 0 + + def _classify_segment(self, seg: str) -> tuple[bool, bool, bool, bool]: + is_paragraph = bool(re.match(r"^\n{2,}$", seg)) + is_sentence = bool(re.match(r"^[!.,;:]+$", seg)) + is_word = bool(re.match(r"^\s+$", seg)) and not is_paragraph + is_content = not is_paragraph and not is_sentence and not is_word + return is_paragraph, is_sentence, is_word, is_content + + def _maybe_update_split_point(self, priority: int): + should_update = ( + self.best_split_idx is None + or priority > self.best_split_idx[2] + or ( + priority == self.best_split_idx[2] + and self.current_length > self.best_split_idx[1] + and self.current_length <= self.max_chars + ) + ) + if should_update: + self.best_split_idx = ( + len(self.current_block) - 1, + self.current_length, + priority, + ) + + def _add_segment(self, seg: str): + self.current_block.append(TextToken(text=seg)) + self.current_length += _grapheme_length(seg) + + def _split_oversized_text(self, seg: str) -> list[list[Token]]: + result: list[list[Token]] = [] + remaining = seg + while remaining: + remaining_len = _grapheme_length(remaining) + if remaining_len <= self.max_chars: + result.append([TextToken(text=remaining)]) + break + chunk_size = self.max_chars - 1 + chunk = grapheme.slice(remaining, 0, chunk_size) + "-" + result.append([TextToken(text=chunk)]) + remaining = grapheme.slice(remaining, chunk_size, remaining_len) + if remaining.startswith(" ") and not remaining.startswith(" "): + remaining = remaining[1:] + return result + + def _strip_leading_space(self): + if self.current_block and isinstance(self.current_block[0], TextToken): + first_token = self.current_block[0] + if first_token.text == " ": + self.current_block.pop(0) + self.current_length -= 1 + elif first_token.text.startswith(" ") and not first_token.text.startswith( + " " + ): + self.current_block[0] = TextToken(text=first_token.text[1:]) + self.current_length -= 1 + + def _split_at_boundary(self): + if self.best_split_idx: + idx, split_length, _ = self.best_split_idx + self.blocks.append(self.current_block[: idx + 1]) + self.current_block = self.current_block[idx + 1 :] + self.current_length = self.current_length - split_length + self.best_split_idx = None + self._strip_leading_space() + return True + elif self.current_block: + self.blocks.append(self.current_block) + self.current_block = [] + self.current_length = 0 + return True + return False + + def _process_text_token(self, token: TextToken): + segments = [s for s in TEXT_SPLITTER.findall(token.text) if s] + + for seg in segments: + seg_len = _grapheme_length(seg) + is_paragraph, is_sentence, is_word, is_content = self._classify_segment(seg) + + while self.current_length + seg_len > self.max_chars: + if self._split_at_boundary(): + pass + else: + if self.current_block: + self.blocks.append(self.current_block) + self.current_block = [] + self.current_length = 0 + for block in self._split_oversized_text(seg): + self.blocks.append(block) + seg = "" + seg_len = 0 + break + + if seg: + self._add_segment(seg) + if is_paragraph: + self._maybe_update_split_point(3) + elif is_sentence: + self._maybe_update_split_point(2) + elif is_word: + self._maybe_update_split_point(1) + elif is_content: + self._maybe_update_split_point(0) + + def _process_token(self, token: Token) -> bool: + if isinstance(token, TextToken): + self._process_text_token(token) + return True + + token_len = self._get_token_length(token) + + if token_len > self.max_chars: + return False + + if self.current_length + token_len > self.max_chars and self.current_block: + self.blocks.append(self.current_block) + self.current_block = [] + self.current_length = 0 + self.best_split_idx = None + + self.current_block.append(token) + self.current_length += token_len + if self.best_split_idx is None or self.best_split_idx[2] < 0: + self.best_split_idx = (len(self.current_block) - 1, self.current_length, 0) + return True + + def split(self, tokens: list[Token]) -> list[list[Token]] | None: + for token in tokens: + if not self._process_token(token): + return None + + self._save_block() + return self.blocks diff --git a/util/util.py b/util/util.py new file mode 100644 --- /dev/null +++ b/util/util.py @@ -0,0 +1,40 @@ +import logging +import os +import sys +from collections.abc import Callable +from typing import Any + +import env + + +shutdown_hook: list[Callable[[], None]] = [] + +logging.basicConfig(stream=sys.stderr, level=logging.DEBUG if env.DEV else logging.INFO) +logging.getLogger("httpx").setLevel(logging.WARNING) +logging.getLogger("httpcore").setLevel(logging.WARNING) +LOGGER = logging.getLogger("XPost") + + +def normalize_service_url(url: str) -> str: + if not url.startswith("https://") and not url.startswith("http://"): + raise ValueError(f"Invalid service url {url}! Only http/https are supported.") + + return url[:-1] if url.endswith("/") else url + + +def read_env(data: dict[str, Any]) -> None: + keys = list(data.keys()) + for key in keys: + val = data[key] + match val: + case str(): + if val.startswith("env:"): + envval = os.environ.get(val[4:]) + if envval is None: + del data[key] + else: + data[key] = envval + case dict(): + read_env(val) + case _: + pass diff --git a/uv.lock b/uv.lock new file mode 100644 --- /dev/null +++ b/uv.lock @@ -0,0 +1,367 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "anyio" +version = "4.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/f0/5eb65b2bb0d09ac6776f2eb54adee6abe8228ea05b20a5ad0e4945de8aac/anyio-4.12.1.tar.gz", hash = "sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703", size = 228685, upload-time = "2026-01-06T11:45:21.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl", hash = "sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c", size = 113592, upload-time = "2026-01-06T11:45:19.497Z" }, +] + +[[package]] +name = "certifi" +version = "2025.10.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/5b/b6ce21586237c77ce67d01dc5507039d444b630dd76611bbca2d8e5dcd91/certifi-2025.10.5.tar.gz", hash = "sha256:47c09d31ccf2acf0be3f701ea53595ee7e0b8fa08801c6624be771df09ae7b43", size = 164519, upload-time = "2025-10-05T04:12:15.808Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/37/af0d2ef3967ac0d6113837b44a4f0bfe1328c2b9763bd5b1744520e5cfed/certifi-2025.10.5-py3-none-any.whl", hash = "sha256:0f212c2744a9bb6de0c56639a6f68afe01ecd92d91f14ae897c4fe7bbeeef0de", size = 163286, upload-time = "2025-10-05T04:12:14.03Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "dnspython" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/8b/57666417c0f90f08bcafa776861060426765fdb422eb10212086fb811d26/dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f", size = 368251, upload-time = "2025-09-07T18:58:00.022Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, +] + +[[package]] +name = "grapheme" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/e7/bbaab0d2a33e07c8278910c1d0d8d4f3781293dfbc70b5c38197159046bf/grapheme-0.6.0.tar.gz", hash = "sha256:44c2b9f21bbe77cfb05835fec230bd435954275267fea1858013b102f8603cca", size = 207306, upload-time = "2020-03-07T17:13:55.492Z" } + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "librt" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/3f/4ca7dd7819bf8ff303aca39c3c60e5320e46e766ab7f7dd627d3b9c11bdf/librt-0.8.0.tar.gz", hash = "sha256:cb74cdcbc0103fc988e04e5c58b0b31e8e5dd2babb9182b6f9490488eb36324b", size = 177306, upload-time = "2026-02-12T14:53:54.743Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/53/f3bc0c4921adb0d4a5afa0656f2c0fbe20e18e3e0295e12985b9a5dc3f55/librt-0.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:17269dd2745dbe8e42475acb28e419ad92dfa38214224b1b01020b8cac70b645", size = 66511, upload-time = "2026-02-12T14:52:30.34Z" }, + { url = "https://files.pythonhosted.org/packages/89/4b/4c96357432007c25a1b5e363045373a6c39481e49f6ba05234bb59a839c1/librt-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f4617cef654fca552f00ce5ffdf4f4b68770f18950e4246ce94629b789b92467", size = 68628, upload-time = "2026-02-12T14:52:31.491Z" }, + { url = "https://files.pythonhosted.org/packages/47/16/52d75374d1012e8fc709216b5eaa25f471370e2a2331b8be00f18670a6c7/librt-0.8.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5cb11061a736a9db45e3c1293cfcb1e3caf205912dfa085734ba750f2197ff9a", size = 198941, upload-time = "2026-02-12T14:52:32.489Z" }, + { url = "https://files.pythonhosted.org/packages/fc/11/d5dd89e5a2228567b1228d8602d896736247424484db086eea6b8010bcba/librt-0.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b4bb00bd71b448f16749909b08a0ff16f58b079e2261c2e1000f2bbb2a4f0a45", size = 210009, upload-time = "2026-02-12T14:52:33.634Z" }, + { url = "https://files.pythonhosted.org/packages/49/d8/fc1a92a77c3020ee08ce2dc48aed4b42ab7c30fb43ce488d388673b0f164/librt-0.8.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:95a719a049f0eefaf1952673223cf00d442952273cbd20cf2ed7ec423a0ef58d", size = 224461, upload-time = "2026-02-12T14:52:34.868Z" }, + { url = "https://files.pythonhosted.org/packages/7f/98/eb923e8b028cece924c246104aa800cf72e02d023a8ad4ca87135b05a2fe/librt-0.8.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bd32add59b58fba3439d48d6f36ac695830388e3da3e92e4fc26d2d02670d19c", size = 217538, upload-time = "2026-02-12T14:52:36.078Z" }, + { url = "https://files.pythonhosted.org/packages/fd/67/24e80ab170674a1d8ee9f9a83081dca4635519dbd0473b8321deecddb5be/librt-0.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4f764b2424cb04524ff7a486b9c391e93f93dc1bd8305b2136d25e582e99aa2f", size = 225110, upload-time = "2026-02-12T14:52:37.301Z" }, + { url = "https://files.pythonhosted.org/packages/d8/c7/6fbdcbd1a6e5243c7989c21d68ab967c153b391351174b4729e359d9977f/librt-0.8.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:f04ca50e847abc486fa8f4107250566441e693779a5374ba211e96e238f298b9", size = 217758, upload-time = "2026-02-12T14:52:38.89Z" }, + { url = "https://files.pythonhosted.org/packages/4b/bd/4d6b36669db086e3d747434430073e14def032dd58ad97959bf7e2d06c67/librt-0.8.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:9ab3a3475a55b89b87ffd7e6665838e8458e0b596c22e0177e0f961434ec474a", size = 218384, upload-time = "2026-02-12T14:52:40.637Z" }, + { url = "https://files.pythonhosted.org/packages/50/2d/afe966beb0a8f179b132f3e95c8dd90738a23e9ebdba10f89a3f192f9366/librt-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3e36a8da17134ffc29373775d88c04832f9ecfab1880470661813e6c7991ef79", size = 241187, upload-time = "2026-02-12T14:52:43.55Z" }, + { url = "https://files.pythonhosted.org/packages/02/d0/6172ea4af2b538462785ab1a68e52d5c99cfb9866a7caf00fdf388299734/librt-0.8.0-cp312-cp312-win32.whl", hash = "sha256:4eb5e06ebcc668677ed6389164f52f13f71737fc8be471101fa8b4ce77baeb0c", size = 54914, upload-time = "2026-02-12T14:52:44.676Z" }, + { url = "https://files.pythonhosted.org/packages/d4/cb/ceb6ed6175612a4337ad49fb01ef594712b934b4bc88ce8a63554832eb44/librt-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:0a33335eb59921e77c9acc05d0e654e4e32e45b014a4d61517897c11591094f8", size = 62020, upload-time = "2026-02-12T14:52:45.676Z" }, + { url = "https://files.pythonhosted.org/packages/f1/7e/61701acbc67da74ce06ddc7ba9483e81c70f44236b2d00f6a4bfee1aacbf/librt-0.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:24a01c13a2a9bdad20997a4443ebe6e329df063d1978bbe2ebbf637878a46d1e", size = 52443, upload-time = "2026-02-12T14:52:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/6d/32/3edb0bcb4113a9c8bdcd1750663a54565d255027657a5df9d90f13ee07fa/librt-0.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7f820210e21e3a8bf8fde2ae3c3d10106d4de9ead28cbfdf6d0f0f41f5b12fa1", size = 66522, upload-time = "2026-02-12T14:52:48.219Z" }, + { url = "https://files.pythonhosted.org/packages/30/ab/e8c3d05e281f5d405ebdcc5bc8ab36df23e1a4b40ac9da8c3eb9928b72b9/librt-0.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4831c44b8919e75ca0dfb52052897c1ef59fdae19d3589893fbd068f1e41afbf", size = 68658, upload-time = "2026-02-12T14:52:50.351Z" }, + { url = "https://files.pythonhosted.org/packages/7c/d3/74a206c47b7748bbc8c43942de3ed67de4c231156e148b4f9250869593df/librt-0.8.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:88c6e75540f1f10f5e0fc5e87b4b6c290f0e90d1db8c6734f670840494764af8", size = 199287, upload-time = "2026-02-12T14:52:51.938Z" }, + { url = "https://files.pythonhosted.org/packages/fa/29/ef98a9131cf12cb95771d24e4c411fda96c89dc78b09c2de4704877ebee4/librt-0.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9646178cd794704d722306c2c920c221abbf080fede3ba539d5afdec16c46dad", size = 210293, upload-time = "2026-02-12T14:52:53.128Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3e/89b4968cb08c53d4c2d8b02517081dfe4b9e07a959ec143d333d76899f6c/librt-0.8.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e1af31a710e17891d9adf0dbd9a5fcd94901a3922a96499abdbf7ce658f4e01", size = 224801, upload-time = "2026-02-12T14:52:54.367Z" }, + { url = "https://files.pythonhosted.org/packages/6d/28/f38526d501f9513f8b48d78e6be4a241e15dd4b000056dc8b3f06ee9ce5d/librt-0.8.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:507e94f4bec00b2f590fbe55f48cd518a208e2474a3b90a60aa8f29136ddbada", size = 218090, upload-time = "2026-02-12T14:52:55.758Z" }, + { url = "https://files.pythonhosted.org/packages/02/ec/64e29887c5009c24dc9c397116c680caffc50286f62bd99c39e3875a2854/librt-0.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f1178e0de0c271231a660fbef9be6acdfa1d596803464706862bef6644cc1cae", size = 225483, upload-time = "2026-02-12T14:52:57.375Z" }, + { url = "https://files.pythonhosted.org/packages/ee/16/7850bdbc9f1a32d3feff2708d90c56fc0490b13f1012e438532781aa598c/librt-0.8.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:71fc517efc14f75c2f74b1f0a5d5eb4a8e06aa135c34d18eaf3522f4a53cd62d", size = 218226, upload-time = "2026-02-12T14:52:58.534Z" }, + { url = "https://files.pythonhosted.org/packages/1c/4a/166bffc992d65ddefa7c47052010a87c059b44a458ebaf8f5eba384b0533/librt-0.8.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:0583aef7e9a720dd40f26a2ad5a1bf2ccbb90059dac2b32ac516df232c701db3", size = 218755, upload-time = "2026-02-12T14:52:59.701Z" }, + { url = "https://files.pythonhosted.org/packages/da/5d/9aeee038bcc72a9cfaaee934463fe9280a73c5440d36bd3175069d2cb97b/librt-0.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5d0f76fc73480d42285c609c0ea74d79856c160fa828ff9aceab574ea4ecfd7b", size = 241617, upload-time = "2026-02-12T14:53:00.966Z" }, + { url = "https://files.pythonhosted.org/packages/64/ff/2bec6b0296b9d0402aa6ec8540aa19ebcb875d669c37800cb43d10d9c3a3/librt-0.8.0-cp313-cp313-win32.whl", hash = "sha256:e79dbc8f57de360f0ed987dc7de7be814b4803ef0e8fc6d3ff86e16798c99935", size = 54966, upload-time = "2026-02-12T14:53:02.042Z" }, + { url = "https://files.pythonhosted.org/packages/08/8d/bf44633b0182996b2c7ea69a03a5c529683fa1f6b8e45c03fe874ff40d56/librt-0.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:25b3e667cbfc9000c4740b282df599ebd91dbdcc1aa6785050e4c1d6be5329ab", size = 62000, upload-time = "2026-02-12T14:53:03.822Z" }, + { url = "https://files.pythonhosted.org/packages/5c/fd/c6472b8e0eac0925001f75e366cf5500bcb975357a65ef1f6b5749389d3a/librt-0.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:e9a3a38eb4134ad33122a6d575e6324831f930a771d951a15ce232e0237412c2", size = 52496, upload-time = "2026-02-12T14:53:04.889Z" }, + { url = "https://files.pythonhosted.org/packages/e0/13/79ebfe30cd273d7c0ce37a5f14dc489c5fb8b722a008983db2cfd57270bb/librt-0.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:421765e8c6b18e64d21c8ead315708a56fc24f44075059702e421d164575fdda", size = 66078, upload-time = "2026-02-12T14:53:06.085Z" }, + { url = "https://files.pythonhosted.org/packages/4b/8f/d11eca40b62a8d5e759239a80636386ef88adecb10d1a050b38cc0da9f9e/librt-0.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:48f84830a8f8ad7918afd743fd7c4eb558728bceab7b0e38fd5a5cf78206a556", size = 68309, upload-time = "2026-02-12T14:53:07.121Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b4/f12ee70a3596db40ff3c88ec9eaa4e323f3b92f77505b4d900746706ec6a/librt-0.8.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9f09d4884f882baa39a7e36bbf3eae124c4ca2a223efb91e567381d1c55c6b06", size = 196804, upload-time = "2026-02-12T14:53:08.164Z" }, + { url = "https://files.pythonhosted.org/packages/8b/7e/70dbbdc0271fd626abe1671ad117bcd61a9a88cdc6a10ccfbfc703db1873/librt-0.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:693697133c3b32aa9b27f040e3691be210e9ac4d905061859a9ed519b1d5a376", size = 206915, upload-time = "2026-02-12T14:53:09.333Z" }, + { url = "https://files.pythonhosted.org/packages/79/13/6b9e05a635d4327608d06b3c1702166e3b3e78315846373446cf90d7b0bf/librt-0.8.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5512aae4648152abaf4d48b59890503fcbe86e85abc12fb9b096fe948bdd816", size = 221200, upload-time = "2026-02-12T14:53:10.68Z" }, + { url = "https://files.pythonhosted.org/packages/35/6c/e19a3ac53e9414de43a73d7507d2d766cd22d8ca763d29a4e072d628db42/librt-0.8.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:995d24caa6bbb34bcdd4a41df98ac6d1af637cfa8975cb0790e47d6623e70e3e", size = 214640, upload-time = "2026-02-12T14:53:12.342Z" }, + { url = "https://files.pythonhosted.org/packages/30/f0/23a78464788619e8c70f090cfd099cce4973eed142c4dccb99fc322283fd/librt-0.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b9aef96d7593584e31ef6ac1eb9775355b0099fee7651fae3a15bc8657b67b52", size = 221980, upload-time = "2026-02-12T14:53:13.603Z" }, + { url = "https://files.pythonhosted.org/packages/03/32/38e21420c5d7aa8a8bd2c7a7d5252ab174a5a8aaec8b5551968979b747bf/librt-0.8.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:4f6e975377fbc4c9567cb33ea9ab826031b6c7ec0515bfae66a4fb110d40d6da", size = 215146, upload-time = "2026-02-12T14:53:14.8Z" }, + { url = "https://files.pythonhosted.org/packages/bb/00/bd9ecf38b1824c25240b3ad982fb62c80f0a969e6679091ba2b3afb2b510/librt-0.8.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:daae5e955764be8fd70a93e9e5133c75297f8bce1e802e1d3683b98f77e1c5ab", size = 215203, upload-time = "2026-02-12T14:53:16.087Z" }, + { url = "https://files.pythonhosted.org/packages/b9/60/7559bcc5279d37810b98d4a52616febd7b8eef04391714fd6bdf629598b1/librt-0.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7bd68cebf3131bb920d5984f75fe302d758db33264e44b45ad139385662d7bc3", size = 237937, upload-time = "2026-02-12T14:53:17.236Z" }, + { url = "https://files.pythonhosted.org/packages/41/cc/be3e7da88f1abbe2642672af1dc00a0bccece11ca60241b1883f3018d8d5/librt-0.8.0-cp314-cp314-win32.whl", hash = "sha256:1e6811cac1dcb27ca4c74e0ca4a5917a8e06db0d8408d30daee3a41724bfde7a", size = 50685, upload-time = "2026-02-12T14:53:18.888Z" }, + { url = "https://files.pythonhosted.org/packages/38/27/e381d0df182a8f61ef1f6025d8b138b3318cc9d18ad4d5f47c3bf7492523/librt-0.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:178707cda89d910c3b28bf5aa5f69d3d4734e0f6ae102f753ad79edef83a83c7", size = 57872, upload-time = "2026-02-12T14:53:19.942Z" }, + { url = "https://files.pythonhosted.org/packages/c5/0c/ca9dfdf00554a44dea7d555001248269a4bab569e1590a91391feb863fa4/librt-0.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:3e8b77b5f54d0937b26512774916041756c9eb3e66f1031971e626eea49d0bf4", size = 48056, upload-time = "2026-02-12T14:53:21.473Z" }, + { url = "https://files.pythonhosted.org/packages/f2/ed/6cc9c4ad24f90c8e782193c7b4a857408fd49540800613d1356c63567d7b/librt-0.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:789911e8fa40a2e82f41120c936b1965f3213c67f5a483fc5a41f5839a05dcbb", size = 68307, upload-time = "2026-02-12T14:53:22.498Z" }, + { url = "https://files.pythonhosted.org/packages/84/d8/0e94292c6b3e00b6eeea39dd44d5703d1ec29b6dafce7eea19dc8f1aedbd/librt-0.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2b37437e7e4ef5e15a297b36ba9e577f73e29564131d86dd75875705e97402b5", size = 70999, upload-time = "2026-02-12T14:53:23.603Z" }, + { url = "https://files.pythonhosted.org/packages/0e/f4/6be1afcbdeedbdbbf54a7c9d73ad43e1bf36897cebf3978308cd64922e02/librt-0.8.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:671a6152edf3b924d98a5ed5e6982ec9cb30894085482acadce0975f031d4c5c", size = 220782, upload-time = "2026-02-12T14:53:25.133Z" }, + { url = "https://files.pythonhosted.org/packages/f0/8d/f306e8caa93cfaf5c6c9e0d940908d75dc6af4fd856baa5535c922ee02b1/librt-0.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8992ca186a1678107b0af3d0c9303d8c7305981b9914989b9788319ed4d89546", size = 235420, upload-time = "2026-02-12T14:53:27.047Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f2/65d86bd462e9c351326564ca805e8457442149f348496e25ccd94583ffa2/librt-0.8.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:001e5330093d887b8b9165823eca6c5c4db183fe4edea4fdc0680bbac5f46944", size = 246452, upload-time = "2026-02-12T14:53:28.341Z" }, + { url = "https://files.pythonhosted.org/packages/03/94/39c88b503b4cb3fcbdeb3caa29672b6b44ebee8dcc8a54d49839ac280f3f/librt-0.8.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d920789eca7ef71df7f31fd547ec0d3002e04d77f30ba6881e08a630e7b2c30e", size = 238891, upload-time = "2026-02-12T14:53:29.625Z" }, + { url = "https://files.pythonhosted.org/packages/e3/c6/6c0d68190893d01b71b9569b07a1c811e280c0065a791249921c83dc0290/librt-0.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:82fb4602d1b3e303a58bfe6165992b5a78d823ec646445356c332cd5f5bbaa61", size = 250249, upload-time = "2026-02-12T14:53:30.93Z" }, + { url = "https://files.pythonhosted.org/packages/52/7a/f715ed9e039035d0ea637579c3c0155ab3709a7046bc408c0fb05d337121/librt-0.8.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:4d3e38797eb482485b486898f89415a6ab163bc291476bd95712e42cf4383c05", size = 240642, upload-time = "2026-02-12T14:53:32.174Z" }, + { url = "https://files.pythonhosted.org/packages/c2/3c/609000a333debf5992efe087edc6467c1fdbdddca5b610355569bbea9589/librt-0.8.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:a905091a13e0884701226860836d0386b88c72ce5c2fdfba6618e14c72be9f25", size = 239621, upload-time = "2026-02-12T14:53:33.39Z" }, + { url = "https://files.pythonhosted.org/packages/b9/df/87b0673d5c395a8f34f38569c116c93142d4dc7e04af2510620772d6bd4f/librt-0.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:375eda7acfce1f15f5ed56cfc960669eefa1ec8732e3e9087c3c4c3f2066759c", size = 262986, upload-time = "2026-02-12T14:53:34.617Z" }, + { url = "https://files.pythonhosted.org/packages/09/7f/6bbbe9dcda649684773aaea78b87fff4d7e59550fbc2877faa83612087a3/librt-0.8.0-cp314-cp314t-win32.whl", hash = "sha256:2ccdd20d9a72c562ffb73098ac411de351b53a6fbb3390903b2d33078ef90447", size = 51328, upload-time = "2026-02-12T14:53:36.15Z" }, + { url = "https://files.pythonhosted.org/packages/bb/f3/e1981ab6fa9b41be0396648b5850267888a752d025313a9e929c4856208e/librt-0.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:25e82d920d4d62ad741592fcf8d0f3bda0e3fc388a184cb7d2f566c681c5f7b9", size = 58719, upload-time = "2026-02-12T14:53:37.183Z" }, + { url = "https://files.pythonhosted.org/packages/94/d1/433b3c06e78f23486fe4fdd19bc134657eb30997d2054b0dbf52bbf3382e/librt-0.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:92249938ab744a5890580d3cb2b22042f0dce71cdaa7c1369823df62bedf7cbc", size = 48753, upload-time = "2026-02-12T14:53:38.539Z" }, +] + +[[package]] +name = "mypy" +version = "1.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/db/4efed9504bc01309ab9c2da7e352cc223569f05478012b5d9ece38fd44d2/mypy-1.19.1.tar.gz", hash = "sha256:19d88bb05303fe63f71dd2c6270daca27cb9401c4ca8255fe50d1d920e0eb9ba", size = 3582404, upload-time = "2025-12-15T05:03:48.42Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/8a/19bfae96f6615aa8a0604915512e0289b1fad33d5909bf7244f02935d33a/mypy-1.19.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a8174a03289288c1f6c46d55cef02379b478bfbc8e358e02047487cad44c6ca1", size = 13206053, upload-time = "2025-12-15T05:03:46.622Z" }, + { url = "https://files.pythonhosted.org/packages/a5/34/3e63879ab041602154ba2a9f99817bb0c85c4df19a23a1443c8986e4d565/mypy-1.19.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffcebe56eb09ff0c0885e750036a095e23793ba6c2e894e7e63f6d89ad51f22e", size = 12219134, upload-time = "2025-12-15T05:03:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/89/cc/2db6f0e95366b630364e09845672dbee0cbf0bbe753a204b29a944967cd9/mypy-1.19.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b64d987153888790bcdb03a6473d321820597ab8dd9243b27a92153c4fa50fd2", size = 12731616, upload-time = "2025-12-15T05:02:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/00/be/dd56c1fd4807bc1eba1cf18b2a850d0de7bacb55e158755eb79f77c41f8e/mypy-1.19.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c35d298c2c4bba75feb2195655dfea8124d855dfd7343bf8b8c055421eaf0cf8", size = 13620847, upload-time = "2025-12-15T05:03:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/6d/42/332951aae42b79329f743bf1da088cd75d8d4d9acc18fbcbd84f26c1af4e/mypy-1.19.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:34c81968774648ab5ac09c29a375fdede03ba253f8f8287847bd480782f73a6a", size = 13834976, upload-time = "2025-12-15T05:03:08.786Z" }, + { url = "https://files.pythonhosted.org/packages/6f/63/e7493e5f90e1e085c562bb06e2eb32cae27c5057b9653348d38b47daaecc/mypy-1.19.1-cp312-cp312-win_amd64.whl", hash = "sha256:b10e7c2cd7870ba4ad9b2d8a6102eb5ffc1f16ca35e3de6bfa390c1113029d13", size = 10118104, upload-time = "2025-12-15T05:03:10.834Z" }, + { url = "https://files.pythonhosted.org/packages/de/9f/a6abae693f7a0c697dbb435aac52e958dc8da44e92e08ba88d2e42326176/mypy-1.19.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e3157c7594ff2ef1634ee058aafc56a82db665c9438fd41b390f3bde1ab12250", size = 13201927, upload-time = "2025-12-15T05:02:29.138Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a4/45c35ccf6e1c65afc23a069f50e2c66f46bd3798cbe0d680c12d12935caa/mypy-1.19.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdb12f69bcc02700c2b47e070238f42cb87f18c0bc1fc4cdb4fb2bc5fd7a3b8b", size = 12206730, upload-time = "2025-12-15T05:03:01.325Z" }, + { url = "https://files.pythonhosted.org/packages/05/bb/cdcf89678e26b187650512620eec8368fded4cfd99cfcb431e4cdfd19dec/mypy-1.19.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f859fb09d9583a985be9a493d5cfc5515b56b08f7447759a0c5deaf68d80506e", size = 12724581, upload-time = "2025-12-15T05:03:20.087Z" }, + { url = "https://files.pythonhosted.org/packages/d1/32/dd260d52babf67bad8e6770f8e1102021877ce0edea106e72df5626bb0ec/mypy-1.19.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9a6538e0415310aad77cb94004ca6482330fece18036b5f360b62c45814c4ef", size = 13616252, upload-time = "2025-12-15T05:02:49.036Z" }, + { url = "https://files.pythonhosted.org/packages/71/d0/5e60a9d2e3bd48432ae2b454b7ef2b62a960ab51292b1eda2a95edd78198/mypy-1.19.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:da4869fc5e7f62a88f3fe0b5c919d1d9f7ea3cef92d3689de2823fd27e40aa75", size = 13840848, upload-time = "2025-12-15T05:02:55.95Z" }, + { url = "https://files.pythonhosted.org/packages/98/76/d32051fa65ecf6cc8c6610956473abdc9b4c43301107476ac03559507843/mypy-1.19.1-cp313-cp313-win_amd64.whl", hash = "sha256:016f2246209095e8eda7538944daa1d60e1e8134d98983b9fc1e92c1fc0cb8dd", size = 10135510, upload-time = "2025-12-15T05:02:58.438Z" }, + { url = "https://files.pythonhosted.org/packages/de/eb/b83e75f4c820c4247a58580ef86fcd35165028f191e7e1ba57128c52782d/mypy-1.19.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06e6170bd5836770e8104c8fdd58e5e725cfeb309f0a6c681a811f557e97eac1", size = 13199744, upload-time = "2025-12-15T05:03:30.823Z" }, + { url = "https://files.pythonhosted.org/packages/94/28/52785ab7bfa165f87fcbb61547a93f98bb20e7f82f90f165a1f69bce7b3d/mypy-1.19.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:804bd67b8054a85447c8954215a906d6eff9cabeabe493fb6334b24f4bfff718", size = 12215815, upload-time = "2025-12-15T05:02:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/0a/c6/bdd60774a0dbfb05122e3e925f2e9e846c009e479dcec4821dad881f5b52/mypy-1.19.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21761006a7f497cb0d4de3d8ef4ca70532256688b0523eee02baf9eec895e27b", size = 12740047, upload-time = "2025-12-15T05:03:33.168Z" }, + { url = "https://files.pythonhosted.org/packages/32/2a/66ba933fe6c76bd40d1fe916a83f04fed253152f451a877520b3c4a5e41e/mypy-1.19.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28902ee51f12e0f19e1e16fbe2f8f06b6637f482c459dd393efddd0ec7f82045", size = 13601998, upload-time = "2025-12-15T05:03:13.056Z" }, + { url = "https://files.pythonhosted.org/packages/e3/da/5055c63e377c5c2418760411fd6a63ee2b96cf95397259038756c042574f/mypy-1.19.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:481daf36a4c443332e2ae9c137dfee878fcea781a2e3f895d54bd3002a900957", size = 13807476, upload-time = "2025-12-15T05:03:17.977Z" }, + { url = "https://files.pythonhosted.org/packages/cd/09/4ebd873390a063176f06b0dbf1f7783dd87bd120eae7727fa4ae4179b685/mypy-1.19.1-cp314-cp314-win_amd64.whl", hash = "sha256:8bb5c6f6d043655e055be9b542aa5f3bdd30e4f3589163e85f93f3640060509f", size = 10281872, upload-time = "2025-12-15T05:03:05.549Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f4/4ce9a05ce5ded1de3ec1c1d96cf9f9504a04e54ce0ed55cfa38619a32b8d/mypy-1.19.1-py3-none-any.whl", hash = "sha256:f1235f5ea01b7db5468d53ece6aaddf1ad0b88d9e7462b86ef96fe04995d7247", size = 2471239, upload-time = "2025-12-15T05:03:07.248Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pathspec" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", size = 131200, upload-time = "2026-01-27T03:59:46.938Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "python-magic" +version = "0.4.27" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/db/0b3e28ac047452d079d375ec6798bf76a036a08182dbb39ed38116a49130/python-magic-0.4.27.tar.gz", hash = "sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b", size = 14677, upload-time = "2022-06-07T20:16:59.508Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/73/9f872cb81fc5c3bb48f7227872c28975f998f3e7c2b1c16e95e6432bbb90/python_magic-0.4.27-py2.py3-none-any.whl", hash = "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3", size = 13840, upload-time = "2022-06-07T20:16:57.763Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/dc/4e6ac71b511b141cf626357a3946679abeba4cf67bc7cc5a17920f31e10d/ruff-0.15.1.tar.gz", hash = "sha256:c590fe13fb57c97141ae975c03a1aedb3d3156030cabd740d6ff0b0d601e203f", size = 4540855, upload-time = "2026-02-12T23:09:09.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/bf/e6e4324238c17f9d9120a9d60aa99a7daaa21204c07fcd84e2ef03bb5fd1/ruff-0.15.1-py3-none-linux_armv6l.whl", hash = "sha256:b101ed7cf4615bda6ffe65bdb59f964e9f4a0d3f85cbf0e54f0ab76d7b90228a", size = 10367819, upload-time = "2026-02-12T23:09:03.598Z" }, + { url = "https://files.pythonhosted.org/packages/b3/ea/c8f89d32e7912269d38c58f3649e453ac32c528f93bb7f4219258be2e7ed/ruff-0.15.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:939c995e9277e63ea632cc8d3fae17aa758526f49a9a850d2e7e758bfef46602", size = 10798618, upload-time = "2026-02-12T23:09:22.928Z" }, + { url = "https://files.pythonhosted.org/packages/5e/0f/1d0d88bc862624247d82c20c10d4c0f6bb2f346559d8af281674cf327f15/ruff-0.15.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1d83466455fdefe60b8d9c8df81d3c1bbb2115cede53549d3b522ce2bc703899", size = 10148518, upload-time = "2026-02-12T23:08:58.339Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c8/291c49cefaa4a9248e986256df2ade7add79388fe179e0691be06fae6f37/ruff-0.15.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9457e3c3291024866222b96108ab2d8265b477e5b1534c7ddb1810904858d16", size = 10518811, upload-time = "2026-02-12T23:09:31.865Z" }, + { url = "https://files.pythonhosted.org/packages/c3/1a/f5707440e5ae43ffa5365cac8bbb91e9665f4a883f560893829cf16a606b/ruff-0.15.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:92c92b003e9d4f7fbd33b1867bb15a1b785b1735069108dfc23821ba045b29bc", size = 10196169, upload-time = "2026-02-12T23:09:17.306Z" }, + { url = "https://files.pythonhosted.org/packages/2a/ff/26ddc8c4da04c8fd3ee65a89c9fb99eaa5c30394269d424461467be2271f/ruff-0.15.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fe5c41ab43e3a06778844c586251eb5a510f67125427625f9eb2b9526535779", size = 10990491, upload-time = "2026-02-12T23:09:25.503Z" }, + { url = "https://files.pythonhosted.org/packages/fc/00/50920cb385b89413f7cdb4bb9bc8fc59c1b0f30028d8bccc294189a54955/ruff-0.15.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:66a6dd6df4d80dc382c6484f8ce1bcceb55c32e9f27a8b94c32f6c7331bf14fb", size = 11843280, upload-time = "2026-02-12T23:09:19.88Z" }, + { url = "https://files.pythonhosted.org/packages/5d/6d/2f5cad8380caf5632a15460c323ae326f1e1a2b5b90a6ee7519017a017ca/ruff-0.15.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6a4a42cbb8af0bda9bcd7606b064d7c0bc311a88d141d02f78920be6acb5aa83", size = 11274336, upload-time = "2026-02-12T23:09:14.907Z" }, + { url = "https://files.pythonhosted.org/packages/a3/1d/5f56cae1d6c40b8a318513599b35ea4b075d7dc1cd1d04449578c29d1d75/ruff-0.15.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ab064052c31dddada35079901592dfba2e05f5b1e43af3954aafcbc1096a5b2", size = 11137288, upload-time = "2026-02-12T23:09:07.475Z" }, + { url = "https://files.pythonhosted.org/packages/cd/20/6f8d7d8f768c93b0382b33b9306b3b999918816da46537d5a61635514635/ruff-0.15.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:5631c940fe9fe91f817a4c2ea4e81f47bee3ca4aa646134a24374f3c19ad9454", size = 11070681, upload-time = "2026-02-12T23:08:55.43Z" }, + { url = "https://files.pythonhosted.org/packages/9a/67/d640ac76069f64cdea59dba02af2e00b1fa30e2103c7f8d049c0cff4cafd/ruff-0.15.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:68138a4ba184b4691ccdc39f7795c66b3c68160c586519e7e8444cf5a53e1b4c", size = 10486401, upload-time = "2026-02-12T23:09:27.927Z" }, + { url = "https://files.pythonhosted.org/packages/65/3d/e1429f64a3ff89297497916b88c32a5cc88eeca7e9c787072d0e7f1d3e1e/ruff-0.15.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:518f9af03bfc33c03bdb4cb63fabc935341bb7f54af500f92ac309ecfbba6330", size = 10197452, upload-time = "2026-02-12T23:09:12.147Z" }, + { url = "https://files.pythonhosted.org/packages/78/83/e2c3bade17dad63bf1e1c2ffaf11490603b760be149e1419b07049b36ef2/ruff-0.15.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:da79f4d6a826caaea95de0237a67e33b81e6ec2e25fc7e1993a4015dffca7c61", size = 10693900, upload-time = "2026-02-12T23:09:34.418Z" }, + { url = "https://files.pythonhosted.org/packages/a1/27/fdc0e11a813e6338e0706e8b39bb7a1d61ea5b36873b351acee7e524a72a/ruff-0.15.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3dd86dccb83cd7d4dcfac303ffc277e6048600dfc22e38158afa208e8bf94a1f", size = 11227302, upload-time = "2026-02-12T23:09:36.536Z" }, + { url = "https://files.pythonhosted.org/packages/f6/58/ac864a75067dcbd3b95be5ab4eb2b601d7fbc3d3d736a27e391a4f92a5c1/ruff-0.15.1-py3-none-win32.whl", hash = "sha256:660975d9cb49b5d5278b12b03bb9951d554543a90b74ed5d366b20e2c57c2098", size = 10462555, upload-time = "2026-02-12T23:09:29.899Z" }, + { url = "https://files.pythonhosted.org/packages/e0/5e/d4ccc8a27ecdb78116feac4935dfc39d1304536f4296168f91ed3ec00cd2/ruff-0.15.1-py3-none-win_amd64.whl", hash = "sha256:c820fef9dd5d4172a6570e5721704a96c6679b80cf7be41659ed439653f62336", size = 11599956, upload-time = "2026-02-12T23:09:01.157Z" }, + { url = "https://files.pythonhosted.org/packages/2a/07/5bda6a85b220c64c65686bc85bd0bbb23b29c62b3a9f9433fa55f17cda93/ruff-0.15.1-py3-none-win_arm64.whl", hash = "sha256:5ff7d5f0f88567850f45081fac8f4ec212be8d0b963e385c3f7d0d2eb4899416", size = 10874604, upload-time = "2026-02-12T23:09:05.515Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "websockets" +version = "15.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/e6/26d09fab466b7ca9c7737474c52be4f76a40301b08362eb2dbc19dcc16c1/websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee", size = 177016, upload-time = "2025-03-05T20:03:41.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/6b/4545a0d843594f5d0771e86463606a3988b5a09ca5123136f8a76580dd63/websockets-15.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3e90baa811a5d73f3ca0bcbf32064d663ed81318ab225ee4f427ad4e26e5aff3", size = 175437, upload-time = "2025-03-05T20:02:16.706Z" }, + { url = "https://files.pythonhosted.org/packages/f4/71/809a0f5f6a06522af902e0f2ea2757f71ead94610010cf570ab5c98e99ed/websockets-15.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:592f1a9fe869c778694f0aa806ba0374e97648ab57936f092fd9d87f8bc03665", size = 173096, upload-time = "2025-03-05T20:02:18.832Z" }, + { url = "https://files.pythonhosted.org/packages/3d/69/1a681dd6f02180916f116894181eab8b2e25b31e484c5d0eae637ec01f7c/websockets-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2", size = 173332, upload-time = "2025-03-05T20:02:20.187Z" }, + { url = "https://files.pythonhosted.org/packages/a6/02/0073b3952f5bce97eafbb35757f8d0d54812b6174ed8dd952aa08429bcc3/websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b56bdcdb4505c8078cb6c7157d9811a85790f2f2b3632c7d1462ab5783d215", size = 183152, upload-time = "2025-03-05T20:02:22.286Z" }, + { url = "https://files.pythonhosted.org/packages/74/45/c205c8480eafd114b428284840da0b1be9ffd0e4f87338dc95dc6ff961a1/websockets-15.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0af68c55afbd5f07986df82831c7bff04846928ea8d1fd7f30052638788bc9b5", size = 182096, upload-time = "2025-03-05T20:02:24.368Z" }, + { url = "https://files.pythonhosted.org/packages/14/8f/aa61f528fba38578ec553c145857a181384c72b98156f858ca5c8e82d9d3/websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dee438fed052b52e4f98f76c5790513235efaa1ef7f3f2192c392cd7c91b65", size = 182523, upload-time = "2025-03-05T20:02:25.669Z" }, + { url = "https://files.pythonhosted.org/packages/ec/6d/0267396610add5bc0d0d3e77f546d4cd287200804fe02323797de77dbce9/websockets-15.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d5f6b181bb38171a8ad1d6aa58a67a6aa9d4b38d0f8c5f496b9e42561dfc62fe", size = 182790, upload-time = "2025-03-05T20:02:26.99Z" }, + { url = "https://files.pythonhosted.org/packages/02/05/c68c5adbf679cf610ae2f74a9b871ae84564462955d991178f95a1ddb7dd/websockets-15.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5d54b09eba2bada6011aea5375542a157637b91029687eb4fdb2dab11059c1b4", size = 182165, upload-time = "2025-03-05T20:02:30.291Z" }, + { url = "https://files.pythonhosted.org/packages/29/93/bb672df7b2f5faac89761cb5fa34f5cec45a4026c383a4b5761c6cea5c16/websockets-15.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be571a8b5afed347da347bfcf27ba12b069d9d7f42cb8c7028b5e98bbb12597", size = 182160, upload-time = "2025-03-05T20:02:31.634Z" }, + { url = "https://files.pythonhosted.org/packages/ff/83/de1f7709376dc3ca9b7eeb4b9a07b4526b14876b6d372a4dc62312bebee0/websockets-15.0.1-cp312-cp312-win32.whl", hash = "sha256:c338ffa0520bdb12fbc527265235639fb76e7bc7faafbb93f6ba80d9c06578a9", size = 176395, upload-time = "2025-03-05T20:02:33.017Z" }, + { url = "https://files.pythonhosted.org/packages/7d/71/abf2ebc3bbfa40f391ce1428c7168fb20582d0ff57019b69ea20fa698043/websockets-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7", size = 176841, upload-time = "2025-03-05T20:02:34.498Z" }, + { url = "https://files.pythonhosted.org/packages/cb/9f/51f0cf64471a9d2b4d0fc6c534f323b664e7095640c34562f5182e5a7195/websockets-15.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee443ef070bb3b6ed74514f5efaa37a252af57c90eb33b956d35c8e9c10a1931", size = 175440, upload-time = "2025-03-05T20:02:36.695Z" }, + { url = "https://files.pythonhosted.org/packages/8a/05/aa116ec9943c718905997412c5989f7ed671bc0188ee2ba89520e8765d7b/websockets-15.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a939de6b7b4e18ca683218320fc67ea886038265fd1ed30173f5ce3f8e85675", size = 173098, upload-time = "2025-03-05T20:02:37.985Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0b/33cef55ff24f2d92924923c99926dcce78e7bd922d649467f0eda8368923/websockets-15.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:746ee8dba912cd6fc889a8147168991d50ed70447bf18bcda7039f7d2e3d9151", size = 173329, upload-time = "2025-03-05T20:02:39.298Z" }, + { url = "https://files.pythonhosted.org/packages/31/1d/063b25dcc01faa8fada1469bdf769de3768b7044eac9d41f734fd7b6ad6d/websockets-15.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:595b6c3969023ecf9041b2936ac3827e4623bfa3ccf007575f04c5a6aa318c22", size = 183111, upload-time = "2025-03-05T20:02:40.595Z" }, + { url = "https://files.pythonhosted.org/packages/93/53/9a87ee494a51bf63e4ec9241c1ccc4f7c2f45fff85d5bde2ff74fcb68b9e/websockets-15.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c714d2fc58b5ca3e285461a4cc0c9a66bd0e24c5da9911e30158286c9b5be7f", size = 182054, upload-time = "2025-03-05T20:02:41.926Z" }, + { url = "https://files.pythonhosted.org/packages/ff/b2/83a6ddf56cdcbad4e3d841fcc55d6ba7d19aeb89c50f24dd7e859ec0805f/websockets-15.0.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f3c1e2ab208db911594ae5b4f79addeb3501604a165019dd221c0bdcabe4db8", size = 182496, upload-time = "2025-03-05T20:02:43.304Z" }, + { url = "https://files.pythonhosted.org/packages/98/41/e7038944ed0abf34c45aa4635ba28136f06052e08fc2168520bb8b25149f/websockets-15.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:229cf1d3ca6c1804400b0a9790dc66528e08a6a1feec0d5040e8b9eb14422375", size = 182829, upload-time = "2025-03-05T20:02:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/e0/17/de15b6158680c7623c6ef0db361da965ab25d813ae54fcfeae2e5b9ef910/websockets-15.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:756c56e867a90fb00177d530dca4b097dd753cde348448a1012ed6c5131f8b7d", size = 182217, upload-time = "2025-03-05T20:02:50.14Z" }, + { url = "https://files.pythonhosted.org/packages/33/2b/1f168cb6041853eef0362fb9554c3824367c5560cbdaad89ac40f8c2edfc/websockets-15.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:558d023b3df0bffe50a04e710bc87742de35060580a293c2a984299ed83bc4e4", size = 182195, upload-time = "2025-03-05T20:02:51.561Z" }, + { url = "https://files.pythonhosted.org/packages/86/eb/20b6cdf273913d0ad05a6a14aed4b9a85591c18a987a3d47f20fa13dcc47/websockets-15.0.1-cp313-cp313-win32.whl", hash = "sha256:ba9e56e8ceeeedb2e080147ba85ffcd5cd0711b89576b83784d8605a7df455fa", size = 176393, upload-time = "2025-03-05T20:02:53.814Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6c/c65773d6cab416a64d191d6ee8a8b1c68a09970ea6909d16965d26bfed1e/websockets-15.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:e09473f095a819042ecb2ab9465aee615bd9c2028e4ef7d933600a8401c79561", size = 176837, upload-time = "2025-03-05T20:02:55.237Z" }, + { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, +] + +[[package]] +name = "xpost" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "dnspython" }, + { name = "grapheme" }, + { name = "httpx" }, + { name = "python-magic" }, + { name = "websockets" }, +] + +[package.dev-dependencies] +dev = [ + { name = "mypy" }, + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "dnspython", specifier = ">=2.8.0" }, + { name = "grapheme", specifier = ">=0.6.0" }, + { name = "httpx", specifier = ">=0.27.0" }, + { name = "python-magic", specifier = ">=0.4.27" }, + { name = "websockets", specifier = ">=15.0.1" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "mypy", specifier = ">=1.15.0" }, + { name = "pytest", specifier = ">=8.4.2" }, + { name = "ruff", specifier = ">=0.9.0" }, +]