diff --git a/.gitea/workflows/publish-ses.yml b/.gitea/workflows/publish-ses.yml index e59119b..5366f98 100644 --- a/.gitea/workflows/publish-ses.yml +++ b/.gitea/workflows/publish-ses.yml @@ -34,7 +34,7 @@ jobs: WIKI_BOT_USER: ${{ secrets.WIKI_BOT_USER }} WIKI_BOT_PASSWORD: ${{ secrets.WIKI_BOT_PASSWORD }} SOURCE_REF: ${{ inputs.SOURCE_REF }} - GITEA_REPO_URL: ${{ vars.URL_TO_GITEA }}/novoyuuparosk-wiki/ses-light-novel + GITEA_REPO_URL: ${{ vars.PUBLIC_URL_TO_GITEA }}/novoyuuparosk-wiki/ses-light-novel run: | python auto-wiki/pipelines/ses/publish.py \ --source-dir ses-light-novel \ diff --git a/.gitea/workflows/publish-songs.yml b/.gitea/workflows/publish-songs.yml index 29b46fe..9cc8a42 100644 --- a/.gitea/workflows/publish-songs.yml +++ b/.gitea/workflows/publish-songs.yml @@ -62,7 +62,7 @@ jobs: WIKI_BOT_USER: ${{ secrets.WIKI_BOT_USER }} WIKI_BOT_PASSWORD: ${{ secrets.WIKI_BOT_PASSWORD }} SOURCE_REF: ${{ inputs.SOURCE_REF }} - GITEA_REPO_URL: ${{ vars.URL_TO_GITEA }}/novoyuuparosk-wiki/ncmr-songs + GITEA_REPO_URL: ${{ vars.PUBLIC_URL_TO_GITEA }}/novoyuuparosk-wiki/ncmr-songs MODE: ${{ steps.diff.outputs.mode }} FILES: ${{ steps.diff.outputs.files }} run: | diff --git a/.gitea/workflows/publish-tech.yml b/.gitea/workflows/publish-tech.yml index 3d91f36..3c27565 100644 --- a/.gitea/workflows/publish-tech.yml +++ b/.gitea/workflows/publish-tech.yml @@ -34,7 +34,7 @@ jobs: WIKI_BOT_USER: ${{ secrets.WIKI_BOT_USER }} WIKI_BOT_PASSWORD: ${{ secrets.WIKI_BOT_PASSWORD }} SOURCE_REF: ${{ inputs.SOURCE_REF }} - GITEA_REPO_URL: ${{ vars.URL_TO_GITEA }}/novoyuuparosk-wiki/tech-blogs + GITEA_REPO_URL: ${{ vars.PUBLIC_URL_TO_GITEA }}/novoyuuparosk-wiki/tech-blogs run: | python auto-wiki/pipelines/tech/publish.py \ --source-dir tech-blogs \ diff --git a/README.md b/README.md index 0bfe6d5..584be18 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,10 @@ Secrets and variables are scoped to the `novoyuuparosk-wiki` org, inherited by a **Variables:** - `WIKI_BASE_URL` = `https://wiki.novoyuuparosk.org` - `WIKI_API_URL` = `https://wiki.novoyuuparosk.org/api.php` -- `URL_TO_GITEA` = Gitea instance base URL (e.g. `http://localhost:3005`). Named with `URL_TO_` prefix — Gitea blocks variable names starting with `GITEA_` or `GITHUB_`. +- `URL_TO_GITEA` = **runner-internal** Gitea base URL, used only for cloning (e.g. `http://localhost:3005` — the runner uses host networking). Named with `URL_TO_` prefix — Gitea blocks variable names starting with `GITEA_` or `GITHUB_`. +- `PUBLIC_URL_TO_GITEA` = **public** Gitea base URL (`https://git.novoyuuparosk.org`), used only for the source link in each published page's banner. + +These two must stay separate. `URL_TO_GITEA` is a loopback address, so composing the banner from it published every page — across all three pipelines — citing `localhost:3005`, which resolves for nobody reading the wiki. `lib/wiki.public_source_base()` now rejects a loopback, mDNS, or private-range value, so that misconfiguration fails the run instead of reaching a page. ## Branch naming @@ -98,6 +101,8 @@ Secrets and variables are scoped to the `novoyuuparosk-wiki` org, inherited by a | MediaWiki API path | `api.php` (classic action API) | 2026-06-09 | | Branch naming (this repo) | `automation/` for pipeline-development branches | 2026-06-09 | | Variable naming | `URL_TO_GITEA` not `GITEA_URL` — Gitea blocks `GITEA_`/`GITHUB_` prefixes | 2026-06-09 | +| **Clone URL ≠ banner URL** | Split into `URL_TO_GITEA` (runner-internal, for cloning) and `PUBLIC_URL_TO_GITEA` (public, for the page banner). One variable served both, and since host networking forces the clone address to `localhost:3005`, every auto-published page in every pipeline cited a link no reader could open. The banner value is validated by `public_source_base()` — loopback, mDNS and private-range addresses abort the run. Pages published before this carry the bad URL in their wikitext and need a full re-publish per pipeline to clear | 2026-09-09 | +| Source URL separators | `rel.as_posix()` in the banner URL, so the path separator never depends on the OS the publisher happens to run on | 2026-09-09 | | Columns shorthand | Side-by-side columns authored as a ` ```columns ` fenced block, expanded post-Pandoc in shared `lib/wiki.py` (universal across pipelines). No wiki template or PHP extension — runs Pi-side before the API call | 2026-06-14 | Per-pipeline decisions live in each pipeline's README. @@ -113,6 +118,7 @@ Via the Gitea web UI logged in as `mikkeli` (an owner of `novoyuuparosk-wiki`): - [x] `WIKI_BASE_URL` - [x] `WIKI_API_URL` - [x] `URL_TO_GITEA` + - [ ] `PUBLIC_URL_TO_GITEA` = `https://git.novoyuuparosk.org` — **required**; the publishers abort without it With Pi access: diff --git a/lib/wiki.py b/lib/wiki.py index 0040678..8ab8645 100644 --- a/lib/wiki.py +++ b/lib/wiki.py @@ -1,6 +1,7 @@ """Shared utilities for novoyuuparosk-auto-wiki pipelines.""" import html +import ipaddress import os import re import subprocess @@ -199,6 +200,63 @@ def markdown_to_wikitext(body: str) -> str: return expand_columns(result.stdout) +_LOOPBACK_HOSTNAMES = frozenset({"localhost", "localhost.localdomain", "ip6-localhost"}) + + +def public_source_base(env_var: str = "GITEA_REPO_URL") -> str: + """The public base URL for the source links baked into published pages. + + This URL ends up in the `{{Auto-generated}}` banner of every page, so it is + read by anyone on the internet — it is *not* the address the runner clones + from. Those two are different hosts here: the runner container uses host + networking and reaches Gitea on `localhost:3005`, while a reader needs the + public name. Composing the banner from the clone address published pages + citing `localhost:3005`, which is why this is validated rather than trusted. + + Rejects an address that cannot possibly work for a reader — unset, missing a + scheme or host, loopback, mDNS (`*.local`), or a private IP range — so the + misconfiguration fails the run instead of being written to the wiki. + """ + raw = os.environ.get(env_var, "").strip().rstrip("/") + if not raw: + raise RuntimeError( + f"{env_var} is unset. It must be the *public* Gitea base URL plus the " + f"repository path (e.g. https://git.example.org/org/repo); the workflow " + f"composes it from the PUBLIC_URL_TO_GITEA Gitea variable." + ) + + parsed = urlparse(raw) + if not parsed.scheme or not parsed.netloc: + raise RuntimeError( + f"{env_var} is not an absolute URL: {raw!r}. Expected scheme and host, " + f"e.g. https://git.example.org/org/repo." + ) + + host = (parsed.hostname or "").lower() + reason = None + if host in _LOOPBACK_HOSTNAMES: + reason = "a loopback hostname" + elif host.endswith(".local"): + reason = "an mDNS name, resolvable only on the local network" + else: + try: + address = ipaddress.ip_address(host) + except ValueError: + address = None + if address is not None and (address.is_loopback or address.is_private): + reason = "a loopback or private-range IP address" + + if reason: + raise RuntimeError( + f"{env_var} points at {reason}: {raw!r}. This URL is published in the " + f"page banner and must resolve for a reader off the network. Set the " + f"PUBLIC_URL_TO_GITEA Gitea variable to the public Gitea base URL — " + f"URL_TO_GITEA is the runner-internal clone address and is not usable here." + ) + + return raw + + def connect_wiki() -> mwclient.Site: api_url = os.environ["WIKI_API_URL"] parsed = urlparse(api_url) diff --git a/pipelines/ses/README.md b/pipelines/ses/README.md index 9fe7747..074e7b6 100644 --- a/pipelines/ses/README.md +++ b/pipelines/ses/README.md @@ -48,10 +48,11 @@ Triggers: | `WIKI_BASE_URL` | variable | Gitea (cross-cutting) | Wiki base URL | | `WIKI_BOT_USER` | secret | Gitea (cross-cutting) | Bot login | | `WIKI_BOT_PASSWORD` | secret | Gitea (cross-cutting) | BotPasswords value | -| `URL_TO_GITEA` | variable | Gitea (cross-cutting) | Gitea instance base URL for cloning | +| `URL_TO_GITEA` | variable | Gitea (cross-cutting) | Runner-internal Gitea base URL, for cloning only | +| `PUBLIC_URL_TO_GITEA` | variable | Gitea (cross-cutting) | Public Gitea base URL, for the banner source link only | | `FAPAT` | secret | Gitea (cross-cutting) | Full-Access PAT for cloning private repos | | `SOURCE_REF` | env (set in workflow) | `github.sha` | Pushed commit SHA, used in banner and edit summary | -| `GITEA_REPO_URL` | env (set in workflow) | composed from vars | Source-file URL base for the banner link | +| `GITEA_REPO_URL` | env (set in workflow) | `PUBLIC_URL_TO_GITEA` + repo path | Source-file URL base for the banner link. Validated by `public_source_base()`: a loopback, mDNS or private-range host aborts the run rather than publishing an unreachable link | ## Dependencies diff --git a/pipelines/ses/publish.py b/pipelines/ses/publish.py index 7f5f2e1..ffef62a 100644 --- a/pipelines/ses/publish.py +++ b/pipelines/ses/publish.py @@ -10,7 +10,12 @@ import frontmatter import mwclient sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) -from lib.wiki import AUTO_BANNER_PREFIX, connect_wiki, markdown_to_wikitext +from lib.wiki import ( + AUTO_BANNER_PREFIX, + connect_wiki, + markdown_to_wikitext, + public_source_base, +) def validate(fm: dict, path: Path) -> None: @@ -59,7 +64,7 @@ def publish_one(abs_path: Path, source_dir: Path, post, site: mwclient.Site, sou page_type = get_page_type(fm, abs_path, source_dir) body_wikitext = markdown_to_wikitext(post.content) - source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel}" + source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel.as_posix()}" page_content = build_wikitext(fm, body_wikitext, source_url, source_ref, page_type) page = site.pages[title] @@ -111,7 +116,7 @@ def main() -> None: source_dir = args.source_dir.resolve() source_ref = os.environ.get("SOURCE_REF", "") - gitea_repo_url = os.environ.get("GITEA_REPO_URL", "").rstrip("/") + gitea_repo_url = public_source_base() if args.all: candidates = list(source_dir.rglob("*.md")) diff --git a/pipelines/songs/README.md b/pipelines/songs/README.md index 1ffea66..11de929 100644 --- a/pipelines/songs/README.md +++ b/pipelines/songs/README.md @@ -53,10 +53,11 @@ Triggers: | `WIKI_BASE_URL` | variable | Gitea (cross-cutting) | Wiki base URL | | `WIKI_BOT_USER` | secret | Gitea (cross-cutting) | Bot login (`Mikkeli@giteaBot`) | | `WIKI_BOT_PASSWORD` | secret | Gitea (cross-cutting) | BotPasswords value | -| `URL_TO_GITEA` | variable | Gitea (cross-cutting) | Gitea instance base URL for cloning | +| `URL_TO_GITEA` | variable | Gitea (cross-cutting) | Runner-internal Gitea base URL, for cloning only | +| `PUBLIC_URL_TO_GITEA` | variable | Gitea (cross-cutting) | Public Gitea base URL, for the banner source link only | | `FAPAT` | secret | Gitea (cross-cutting) | Full-Access PAT for cloning private repos | | `SOURCE_REF` | env (set in workflow) | `github.sha` | Pushed commit SHA, used in banner and edit summary | -| `GITEA_REPO_URL` | env (set in workflow) | composed from vars | Source-file URL base for the banner link | +| `GITEA_REPO_URL` | env (set in workflow) | `PUBLIC_URL_TO_GITEA` + repo path | Source-file URL base for the banner link. Validated by `public_source_base()`: a loopback, mDNS or private-range host aborts the run rather than publishing an unreachable link | ## Dependencies diff --git a/pipelines/songs/publish.py b/pipelines/songs/publish.py index ff81f35..c533a99 100644 --- a/pipelines/songs/publish.py +++ b/pipelines/songs/publish.py @@ -19,6 +19,7 @@ from lib.wiki import ( connect_wiki, extract_md_link_targets, markdown_to_wikitext, + public_source_base, resolve_md_links, strip_first_h1, ) @@ -153,7 +154,7 @@ def render_page(abs_path: Path, source_dir: Path, post, source_ref: str, gitea_r body_wikitext = markdown_to_wikitext(strip_first_h1(post.content)) body_wikitext = resolve_md_links(body_wikitext, abs_path, title_index) - source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel}" + source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel.as_posix()}" lrc_filename = f"{title}.lrc" if fm.get("lrc") else None return build_wikitext(fm, body_wikitext, source_url, source_ref, lrc_filename) @@ -241,7 +242,7 @@ def main() -> None: source_dir = args.source_dir.resolve() source_ref = os.environ.get("SOURCE_REF", "") - gitea_repo_url = os.environ.get("GITEA_REPO_URL", "").rstrip("/") + gitea_repo_url = public_source_base() # The repository is always scanned in full, whatever gets published: link # resolution and title-collision checks must see every publishable file, or diff --git a/pipelines/tech/README.md b/pipelines/tech/README.md index 4efefb4..4e51bde 100644 --- a/pipelines/tech/README.md +++ b/pipelines/tech/README.md @@ -42,10 +42,11 @@ Triggers: | `WIKI_BASE_URL` | variable | Gitea (cross-cutting) | Wiki base URL | | `WIKI_BOT_USER` | secret | Gitea (cross-cutting) | Bot login | | `WIKI_BOT_PASSWORD` | secret | Gitea (cross-cutting) | BotPasswords value | -| `URL_TO_GITEA` | variable | Gitea (cross-cutting) | Gitea instance base URL for cloning | +| `URL_TO_GITEA` | variable | Gitea (cross-cutting) | Runner-internal Gitea base URL, for cloning only | +| `PUBLIC_URL_TO_GITEA` | variable | Gitea (cross-cutting) | Public Gitea base URL, for the banner source link only | | `FAPAT` | secret | Gitea (cross-cutting) | Full-Access PAT for cloning private repos | | `SOURCE_REF` | env (set in workflow) | `github.sha` | Pushed commit SHA, used in banner and edit summary | -| `GITEA_REPO_URL` | env (set in workflow) | composed from vars | Source-file URL base for the banner link | +| `GITEA_REPO_URL` | env (set in workflow) | `PUBLIC_URL_TO_GITEA` + repo path | Source-file URL base for the banner link. Validated by `public_source_base()`: a loopback, mDNS or private-range host aborts the run rather than publishing an unreachable link | ## Dependencies diff --git a/pipelines/tech/publish.py b/pipelines/tech/publish.py index 6ba6091..e79b09e 100644 --- a/pipelines/tech/publish.py +++ b/pipelines/tech/publish.py @@ -11,7 +11,12 @@ import frontmatter import mwclient sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) -from lib.wiki import AUTO_BANNER_PREFIX, connect_wiki, markdown_to_wikitext +from lib.wiki import ( + AUTO_BANNER_PREFIX, + connect_wiki, + markdown_to_wikitext, + public_source_base, +) def validate(fm: dict, path: Path) -> None: @@ -58,7 +63,7 @@ def publish_one(abs_path: Path, source_dir: Path, post, site: mwclient.Site, sou title = str(fm["title"]) body_wikitext = markdown_to_wikitext(post.content) - source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel}" + source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel.as_posix()}" page_content = build_wikitext(fm, body_wikitext, source_url, source_ref) page = site.pages[title] @@ -110,7 +115,7 @@ def main() -> None: source_dir = args.source_dir.resolve() source_ref = os.environ.get("SOURCE_REF", "") - gitea_repo_url = os.environ.get("GITEA_REPO_URL", "").rstrip("/") + gitea_repo_url = public_source_base() if args.all: candidates = list(source_dir.rglob("*.md")) diff --git a/tests/test_wiki.py b/tests/test_wiki.py index 37f5879..702af57 100644 --- a/tests/test_wiki.py +++ b/tests/test_wiki.py @@ -15,6 +15,7 @@ from lib.wiki import ( expand_columns, extract_md_link_targets, markdown_to_wikitext, + public_source_base, resolve_md_links, strip_first_h1, ) @@ -214,3 +215,72 @@ class TestPandocContract: wikitext = markdown_to_wikitext("```columns\n**A**\nl1\n===\n**B**\nl2\n```") assert wikitext.count("") == 2 assert "'''A'''" in wikitext + + +class TestPublicSourceBase: + """The banner URL is read by the public, so a runner-only address must fail. + + Every one of these rejections was a page published with a link to + localhost:3005, which is what the whole guard exists to prevent. + """ + + def test_accepts_a_public_https_url(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "https://git.novoyuuparosk.org/org/repo") + assert public_source_base() == "https://git.novoyuuparosk.org/org/repo" + + def test_accepts_a_public_host_with_a_port(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "http://git.novoyuuparosk.org:1551/org/repo") + assert public_source_base() == "http://git.novoyuuparosk.org:1551/org/repo" + + def test_strips_a_trailing_slash(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "https://git.example.org/org/repo/") + assert public_source_base() == "https://git.example.org/org/repo" + + def test_strips_surrounding_whitespace(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", " https://git.example.org/org/repo ") + assert public_source_base() == "https://git.example.org/org/repo" + + def test_unset_is_rejected(self, monkeypatch): + monkeypatch.delenv("GITEA_REPO_URL", raising=False) + with pytest.raises(RuntimeError, match="is unset"): + public_source_base() + + def test_empty_is_rejected(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "") + with pytest.raises(RuntimeError, match="is unset"): + public_source_base() + + def test_a_bare_path_is_rejected(self, monkeypatch): + # What an unset PUBLIC_URL_TO_GITEA leaves the workflow composing. + monkeypatch.setenv("GITEA_REPO_URL", "/novoyuuparosk-wiki/ncmr-songs") + with pytest.raises(RuntimeError, match="not an absolute URL"): + public_source_base() + + def test_localhost_is_rejected(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "http://localhost:3005/org/repo") + with pytest.raises(RuntimeError, match="loopback hostname"): + public_source_base() + + def test_loopback_ip_is_rejected(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "http://127.0.0.1:3005/org/repo") + with pytest.raises(RuntimeError, match="loopback or private-range"): + public_source_base() + + def test_private_range_ip_is_rejected(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "http://192.168.1.20:3005/org/repo") + with pytest.raises(RuntimeError, match="loopback or private-range"): + public_source_base() + + def test_mdns_name_is_rejected(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "http://pi5-16.local:3005/org/repo") + with pytest.raises(RuntimeError, match="mDNS"): + public_source_base() + + def test_rejection_names_the_variable_to_set(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "http://localhost:3005/org/repo") + with pytest.raises(RuntimeError, match="PUBLIC_URL_TO_GITEA"): + public_source_base() + + def test_reads_the_named_variable(self, monkeypatch): + monkeypatch.setenv("OTHER_BASE", "https://git.example.org/org/repo") + assert public_source_base("OTHER_BASE") == "https://git.example.org/org/repo"