novoyuuparosk-auto-wiki/lib/wiki.py

"""Shared utilities for novoyuuparosk-auto-wiki pipelines."""

import html
import ipaddress
import os
import re
import subprocess
import urllib.parse
from pathlib import Path
from urllib.parse import urlparse

import mwclient

AUTO_BANNER_PREFIX = "{{Auto-generated"

# A ```columns fenced block is passed through Pandoc verbatim as
# <pre class="columns">…</pre>; columns within it are separated by a line of ===.
_COLUMNS_BLOCK_RE = re.compile(r'<pre class="columns">(.*?)</pre>', re.DOTALL)
_COLUMN_SEP_RE = re.compile(r"^\s*===\s*$", re.MULTILINE)

# Minimal Markdown emphasis → wikitext, for use inside a verbatim columns block
# (Pandoc doesn't process Markdown there). Asterisk style only, single line.
# Order matters: bold-italic (***) before bold (**) before italic (*).
_BOLD_ITALIC_RE = re.compile(r"\*\*\*(.+?)\*\*\*")
_BOLD_RE = re.compile(r"\*\*(.+?)\*\*")
_ITALIC_RE = re.compile(r"\*(.+?)\*")

# Pandoc's mediawiki writer renders a *relative* Markdown link target as a
# wikilink carrying the raw path — [label](notes.md) becomes [[notes.md|label]] —
# and an absolute URL as [url label]. Both shapes are matched here so that a
# pandoc version which classifies relative targets the other way still resolves.
# The alternation is scanned left to right, so a [[...]] is consumed whole and
# the single-bracket branch never re-matches its interior.
_MD_LINK_RE = re.compile(
    r"\[\[(?P<wiki_target>[^\[\]|]+?)(?:\|(?P<wiki_label>[^\[\]]*))?\]\]"
    r"|\[(?P<ext_target>[^\[\]\s|]+)(?:[ \t]+(?P<ext_label>[^\[\]]*))?\]"
)

# Inline Markdown link target, read from the *source* body before Pandoc runs:
# the "…](target)" tail of [label](target). Used for the link graph only.
_MD_SOURCE_LINK_RE = re.compile(r"\]\(\s*<?([^()<>\s]+)>?")

# Anything still naming a .md file in a link position after resolution means the
# renderer failed to understand it. That aborts the run; it never reaches the wiki.
_UNRESOLVED_LINK_RES = (
    (
        re.compile(r"\[\[[^\[\]]*\.md(?:#[^\[\]]*)?(?:\|[^\[\]]*)?\]\]", re.IGNORECASE),
        "unresolved wikilink",
    ),
    (
        re.compile(r"\[[^\[\]\s|]*\.md[^\[\]]*\]", re.IGNORECASE),
        "unresolved external-style link",
    ),
    (
        re.compile(r"\]\([^()\s]*\.md[^()]*\)", re.IGNORECASE),
        "raw Markdown link (Markdown is not processed inside a ```columns fence)",
    ),
)


def _md_emphasis_to_wikitext(text: str) -> str:
    text = _BOLD_ITALIC_RE.sub(r"'''''\1'''''", text)
    text = _BOLD_RE.sub(r"'''\1'''", text)
    text = _ITALIC_RE.sub(r"''\1''", text)
    return text


def strip_first_h1(text: str) -> str:
    """Remove the first '# Heading' line and any immediately following blank line."""
    lines = text.split("\n")
    for i, line in enumerate(lines):
        if line.strip().startswith("# "):
            del lines[i]
            if i < len(lines) and lines[i].strip() == "":
                del lines[i]
            break
    return "\n".join(lines)


def expand_columns(wikitext: str) -> str:
    """Expand ```columns fenced blocks into a flex row of <poem> columns.

    Authors write a fenced code block tagged ``columns``; Pandoc passes its body
    through verbatim as ``<pre class="columns">…</pre>`` (line breaks and blank
    lines preserved, inline markup entity-escaped). Columns within the block are
    separated by a line containing only ``===``. Each column is wrapped in
    <poem> so its line breaks survive MediaWiki parsing. Content is
    HTML-unescaped (so inline markup such as <b>…</b> renders rather than
    appearing as literal text) and Markdown emphasis (``*italic*``, ``**bold**``,
    ``***bold-italic***``) is converted to wikitext, since Pandoc does not
    process Markdown inside the verbatim block.
    """

    def render(match: re.Match) -> str:
        body = _md_emphasis_to_wikitext(html.unescape(match.group(1)))
        columns = _COLUMN_SEP_RE.split(body)
        poems = "".join("<poem>\n" + col.strip("\n") + "\n</poem>" for col in columns)
        return '<div style="display:flex; gap:3em; align-items:flex-start">' + poems + "</div>"

    return _COLUMNS_BLOCK_RE.sub(render, wikitext)


def extract_md_link_targets(markdown_body: str) -> list[str]:
    """Raw targets of inline Markdown links in an unrendered body.

    Used to build the link graph *before* Pandoc runs, so the publisher can work
    out which pages a changed page points at (and which point back) without
    rendering the whole repository. Targets are returned verbatim, including any
    ``#fragment`` and percent-encoding; the caller resolves them.
    """
    return _MD_SOURCE_LINK_RE.findall(markdown_body)


def assert_no_unresolved_md_links(wikitext: str, source_path: Path) -> None:
    """Abort if any link still names a .md file after resolution.

    Guards against a link shape the resolver does not recognise: a pandoc version
    that writes relative targets differently, or a Markdown link buried in a
    verbatim ```columns fence. Without this, such a link would reach the wiki as
    literal text, silently corrupting a published page.
    """
    for pattern, kind in _UNRESOLVED_LINK_RES:
        found = pattern.search(wikitext)
        if found:
            raise ValueError(
                f"{source_path}: {kind} still references a .md file after "
                f"resolution: {found.group(0)!r}"
            )


def resolve_md_links(wikitext: str, source_path: Path, title_index: dict[str, str]) -> str:
    """Rewrite links to sibling ``.md`` files into wikilinks to their wiki pages.

    A related co-publication is nominated with an ordinary relative Markdown
    link — ``see [dev notes](pulse_dev.md)`` — which stays a working link in
    Gitea's blob view and in any Markdown preview. Here the ``.md`` target is
    resolved against ``title_index`` (absolute path string → wiki page title) and
    replaced with ``[[Wiki page title|label]]``.

    ``title_index`` must cover every publishable file in the repository, not only
    the batch being published, or the same link resolves differently depending on
    which files a push happened to touch.

    A ``.md`` target that cannot be resolved raises ``ValueError``: either the
    file does not exist, or it exists but is not published. Both are authoring
    mistakes, and both fail the run rather than silently degrade the page. Link
    targets that do not name a ``.md`` file are left exactly as Pandoc wrote them.
    """

    def resolve(match: re.Match) -> str:
        groups = match.groupdict()
        if groups["wiki_target"] is not None:
            target, label = groups["wiki_target"], groups["wiki_label"]
        else:
            target, label = groups["ext_target"], groups["ext_label"]

        raw_path, _, fragment = target.partition("#")
        rel_path = urllib.parse.unquote(raw_path)
        if not rel_path.lower().endswith(".md"):
            return match.group(0)

        resolved = (source_path.parent / rel_path).resolve()
        title = title_index.get(str(resolved))
        if title is None:
            reason = (
                "file exists but is not published (no 'wiki.publish: true', "
                "or it lives under wip/)"
                if resolved.exists()
                else "file does not exist"
            )
            raise ValueError(
                f"{source_path}: link to '{target}' cannot be resolved - {reason}"
            )

        anchor = f"#{fragment}" if fragment else ""
        # A label that repeats the path (pandoc's form for an unlabelled link) or
        # repeats the page title needs no pipe: [[Title]] already displays it.
        if label is None or label.strip() in (target, title):
            return f"[[{title}{anchor}]]"
        return f"[[{title}{anchor}|{label}]]"

    resolved = _MD_LINK_RE.sub(resolve, wikitext)
    assert_no_unresolved_md_links(resolved, source_path)
    return resolved


def markdown_to_wikitext(body: str) -> str:
    # encoding is pinned rather than left to text=True, which would decode
    # pandoc's output with the ambient locale encoding. The lyrics are Japanese
    # and Chinese; on any non-UTF-8 locale that decode dies inside subprocess's
    # reader thread, stdout comes back None, and the real error is lost.
    result = subprocess.run(
        ["pandoc", "-f", "markdown", "-t", "mediawiki"],
        input=body,
        capture_output=True,
        encoding="utf-8",
    )
    if result.returncode != 0:
        raise RuntimeError(f"pandoc failed: {result.stderr.strip()}")
    return expand_columns(result.stdout)


_LOOPBACK_HOSTNAMES = frozenset({"localhost", "localhost.localdomain", "ip6-localhost"})


def public_source_base(env_var: str = "GITEA_REPO_URL") -> str:
    """The public base URL for the source links baked into published pages.

    This URL ends up in the `{{Auto-generated}}` banner of every page, so it is
    read by anyone on the internet — it is *not* the address the runner clones
    from. Those two are different hosts here: the runner container uses host
    networking and reaches Gitea on `localhost:3005`, while a reader needs the
    public name. Composing the banner from the clone address published pages
    citing `localhost:3005`, which is why this is validated rather than trusted.

    Rejects an address that cannot possibly work for a reader — unset, missing a
    scheme or host, loopback, mDNS (`*.local`), or a private IP range — so the
    misconfiguration fails the run instead of being written to the wiki.
    """
    raw = os.environ.get(env_var, "").strip().rstrip("/")
    if not raw:
        raise RuntimeError(
            f"{env_var} is unset. It must be the *public* Gitea base URL plus the "
            f"repository path (e.g. https://git.example.org/org/repo); the workflow "
            f"composes it from the PUBLIC_URL_TO_GITEA Gitea variable."
        )

    parsed = urlparse(raw)
    if not parsed.scheme or not parsed.netloc:
        raise RuntimeError(
            f"{env_var} is not an absolute URL: {raw!r}. Expected scheme and host, "
            f"e.g. https://git.example.org/org/repo."
        )

    host = (parsed.hostname or "").lower()
    reason = None
    if host in _LOOPBACK_HOSTNAMES:
        reason = "a loopback hostname"
    elif host.endswith(".local"):
        reason = "an mDNS name, resolvable only on the local network"
    else:
        try:
            address = ipaddress.ip_address(host)
        except ValueError:
            address = None
        if address is not None and (address.is_loopback or address.is_private):
            reason = "a loopback or private-range IP address"

    if reason:
        raise RuntimeError(
            f"{env_var} points at {reason}: {raw!r}. This URL is published in the "
            f"page banner and must resolve for a reader off the network. Set the "
            f"PUBLIC_URL_TO_GITEA Gitea variable to the public Gitea base URL — "
            f"URL_TO_GITEA is the runner-internal clone address and is not usable here."
        )

    return raw


def connect_wiki() -> mwclient.Site:
    api_url = os.environ["WIKI_API_URL"]
    parsed = urlparse(api_url)
    host = parsed.netloc
    path = parsed.path[: parsed.path.rfind("/") + 1] or "/"
    site = mwclient.Site(host, path=path, scheme=parsed.scheme)
    site.login(os.environ["WIKI_BOT_USER"], os.environ["WIKI_BOT_PASSWORD"])
    return site