From 73e5cd7202308e219a00955cad0017747381f709 Mon Sep 17 00:00:00 2001 From: Leander Furumo Date: Thu, 29 Oct 2026 22:24:33 +0100 Subject: [PATCH] render viteboka articles from html instead of stripping wikitext Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 1 - src/assets/viteboka.lua | 21 ++++++++ src/cogs/viteboka.py | 111 ++++++++++++++++++++++++++++++++-------- uv.lock | 2 - 4 files changed, 111 insertions(+), 24 deletions(-) create mode 100644 src/assets/viteboka.lua diff --git a/pyproject.toml b/pyproject.toml index dbb370a..78b020c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,6 @@ dependencies = [ "psycopg[binary]==3.3.*", "pypandoc==1.17.*", "pyyaml==6.0.*", - "regex==2026.9.29", "tzdata", # This only needed if run on windows. I haven't set a version either to ensure it's up to date. "wordcloud==1.9.*", "yt-dlp==2026.10.21", diff --git a/src/assets/viteboka.lua b/src/assets/viteboka.lua new file mode 100644 index 0000000..1d3d8ad --- /dev/null +++ b/src/assets/viteboka.lua @@ -0,0 +1,21 @@ +-- Pandoc filter that cleans up Viteboka article HTML for Discord embeds + +local base_url = "https://viteboka.studentersamfundet.no" + +function Table() return {} end +function Figure() return {} end +function Image() return {} end +function LineBreak() return pandoc.Space() end + +function Link(link) + -- Links to pages that don't exist, and links that only wrapped a removed image + if link.classes:includes("new") or #link.content == 0 then + return link.content + end + + if link.target:sub(1, 1) == "/" then + link.target = base_url .. link.target + end + link.title = "" + return link +end diff --git a/src/cogs/viteboka.py b/src/cogs/viteboka.py index 3fd6c6a..682d702 100644 --- a/src/cogs/viteboka.py +++ b/src/cogs/viteboka.py @@ -1,7 +1,9 @@ +import asyncio +import re + import aiohttp import discord import pypandoc -import regex from discord import app_commands from discord.ext import commands @@ -9,6 +11,8 @@ from cogs.utils import embed_templates WIKI_BASE_URL = "https://viteboka.studentersamfundet.no" API_URL = f"{WIKI_BASE_URL}/w/api.php" +PANDOC_FILTER = "./src/assets/viteboka.lua" +MAX_TEXT_LENGTH = 1000 async def fetch_article(session: aiohttp.ClientSession, title: str): @@ -25,11 +29,18 @@ async def fetch_article(session: aiohttp.ClientSession, title: str): tuple: The title, url, text and image of the article """ + # The wiki renders the article to HTML for us, so templates are already expanded params = { "action": "parse", "format": "json", + "formatversion": 2, "page": title, - "prop": "text|images|displaytitle|wikitext", + "prop": "text", + "section": 0, # Only the introduction + "redirects": 1, + "disableeditsection": 1, + "disabletoc": 1, + "disablelimitreport": 1, } async with session.get(API_URL, params=params) as response: @@ -41,27 +52,85 @@ async def fetch_article(session: aiohttp.ClientSession, title: str): "Fant ingen artikkel med det navnet. Kan hende den finnes, men wiki-søk er balle. De burde tatt søketek" ) - title = parse.get("title") - url = f"{WIKI_BASE_URL}/?curid={parse.get('pageid')}" - image = f"{WIKI_BASE_URL}/w/images/{parse.get('images')[0]}" if parse.get("images") else None - text = parse.get("wikitext")["*"] + title = parse["title"] + url = f"{WIKI_BASE_URL}/?curid={parse['pageid']}" + image = await fetch_page_image(session, title) + text = await asyncio.to_thread( + pypandoc.convert_text, + parse["text"], + "gfm-raw_html", + format="html", + extra_args=["--wrap=none"], + filters=[PANDOC_FILTER], + ) - # If you're reading this, know that I'm on my way - # to your house right now to erase your memories of ever reading this shit. - # Parsing text fucking sucks. Even more so when it's wiki text with no consistencies whatsoever - # This is wildly inefficent but I don't. fucking. care. Just make it work - text = regex.sub( - r"(?=\{)(\{([^{}]|(?1))*\})|\[\[(Kategori|Fil):.+?\]\]|\{.+?\}", "", text, flags=regex.MULTILINE | regex.DOTALL - ) # Remove templates, categories and images - text = pypandoc.convert_text(text, "markdown", format="mediawiki") - text = regex.sub( - r"\[(.+?)\]\(.+? \"wikilink\"\)", r"\1", text, flags=regex.MULTILINE | regex.DOTALL - ) # Pandoc adds alt text to links. discord doesn't support that - text = regex.sub(r"\{.+\}", "", text) # Pandoc adds section links. Remove them - text = text.strip("\n") - text = text[:1000] + "..." if len(text) > 1000 else text + return title, url, shorten(text.strip(), MAX_TEXT_LENGTH), image - return title, url, text, image + +async def fetch_page_image(session: aiohttp.ClientSession, title: str) -> str | None: + """ + Fetch the main image of an article from Viteboka + + Parameters + ---------- + session (aiohttp.ClientSession): The HTTP session to use + title (str): The title of the article + + Returns + ---------- + str | None: The url of the image. None if the article has no image or it couldn't be fetched + """ + + params = { + "action": "query", + "format": "json", + "formatversion": 2, + "titles": title, + "prop": "pageimages", + "piprop": "original", + } + + async with session.get(API_URL, params=params) as response: + if response.status != 200: + return None + pages = (await response.json(content_type=None))["query"]["pages"] + + return pages[0].get("original", {}).get("source") + + +def shorten(text: str, limit: int) -> str: + """ + Shorten markdown text to fit within a limit. + Pandoc puts every paragraph and list item on its own line, so whole lines are kept where possible + + Parameters + ---------- + text (str): The markdown text + limit (int): The maximum length + + Returns + ---------- + str: The shortened text + """ + + if len(text) <= limit: + return text + + shortened = "" + for line in text.split("\n"): + if len(shortened) + len(line) + len("\n...") > limit: + break + shortened += line + "\n" + + room = limit - len(shortened) - len("...") + if room < 100 and shortened.strip(): + return shortened.rstrip() + "\n..." + + # Fill the rest with the start of the line that didn't fit. Its links are removed so cutting it can't break one + line = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", line) + if len(line) <= room: + return shortened + line + "\n..." + return shortened + line[:room].rsplit(" ", 1)[0] + "..." def viteboka_embed(title: str, url: str, text: str, image: str): diff --git a/uv.lock b/uv.lock index fb81ddc..4528f2d 100644 --- a/uv.lock +++ b/uv.lock @@ -1011,7 +1011,6 @@ dependencies = [ { name = "psycopg-pool" }, { name = "pypandoc" }, { name = "pyyaml" }, - { name = "regex" }, { name = "tzdata" }, { name = "wordcloud" }, { name = "yt-dlp" }, @@ -1040,7 +1039,6 @@ requires-dist = [ { name = "psycopg-pool", specifier = "==3.3.*" }, { name = "pypandoc", specifier = "==1.17.*" }, { name = "pyyaml", specifier = "==6.0.*" }, - { name = "regex", specifier = "==2026.9.29" }, { name = "tzdata" }, { name = "wordcloud", specifier = "==1.9.*" }, { name = "yt-dlp", specifier = "==2026.10.21" },