render viteboka articles from html instead of stripping wikitext

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

pyproject.toml

@@ -21,7 +21,6 @@ dependencies = [
     "psycopg[binary]==3.3.*",
     "pypandoc==1.17.*",
     "pyyaml==6.0.*",
-    "regex==2026.9.29",
     "tzdata",  # This only needed if run on windows. I haven't set a version either to ensure it's up to date.
     "wordcloud==1.9.*",
     "yt-dlp==2026.10.21",

src/assets/viteboka.lua

@@ -0,0 +1,21 @@
+-- Pandoc filter that cleans up Viteboka article HTML for Discord embeds
+
+local base_url = "https://viteboka.studentersamfundet.no"
+
+function Table() return {} end
+function Figure() return {} end
+function Image() return {} end
+function LineBreak() return pandoc.Space() end
+
+function Link(link)
+  -- Links to pages that don't exist, and links that only wrapped a removed image
+  if link.classes:includes("new") or #link.content == 0 then
+    return link.content
+  end
+
+  if link.target:sub(1, 1) == "/" then
+    link.target = base_url .. link.target
+  end
+  link.title = ""
+  return link
+end

src/cogs/viteboka.py

@@ -1,7 +1,9 @@
+import asyncio
+import re
+
 import aiohttp
 import discord
 import pypandoc
-import regex
 from discord import app_commands
 from discord.ext import commands
 
@@ -9,6 +11,8 @@ from cogs.utils import embed_templates
 
 WIKI_BASE_URL = "https://viteboka.studentersamfundet.no"
 API_URL = f"{WIKI_BASE_URL}/w/api.php"
+PANDOC_FILTER = "./src/assets/viteboka.lua"
+MAX_TEXT_LENGTH = 1000
 
 
 async def fetch_article(session: aiohttp.ClientSession, title: str):
@@ -25,11 +29,18 @@ async def fetch_article(session: aiohttp.ClientSession, title: str):
     tuple: The title, url, text and image of the article
     """
 
+    # The wiki renders the article to HTML for us, so templates are already expanded
     params = {
         "action": "parse",
         "format": "json",
+        "formatversion": 2,
         "page": title,
-        "prop": "text|images|displaytitle|wikitext",
+        "prop": "text",
+        "section": 0,  # Only the introduction
+        "redirects": 1,
+        "disableeditsection": 1,
+        "disabletoc": 1,
+        "disablelimitreport": 1,
     }
 
     async with session.get(API_URL, params=params) as response:
@@ -41,27 +52,85 @@ async def fetch_article(session: aiohttp.ClientSession, title: str):
             "Fant ingen artikkel med det navnet. Kan hende den finnes, men wiki-søk er balle. De burde tatt søketek"
         )
 
-    title = parse.get("title")
+    title = parse["title"]
-    url = f"{WIKI_BASE_URL}/?curid={parse.get('pageid')}"
+    url = f"{WIKI_BASE_URL}/?curid={parse['pageid']}"
-    image = f"{WIKI_BASE_URL}/w/images/{parse.get('images')[0]}" if parse.get("images") else None
+    image = await fetch_page_image(session, title)
-    text = parse.get("wikitext")["*"]
+    text = await asyncio.to_thread(
+        pypandoc.convert_text,
+        parse["text"],
+        "gfm-raw_html",
+        format="html",
+        extra_args=["--wrap=none"],
+        filters=[PANDOC_FILTER],
+    )
 
-    # If you're reading this, know that I'm on my way
+    return title, url, shorten(text.strip(), MAX_TEXT_LENGTH), image
-    # to your house right now to erase your memories of ever reading this shit.
-    # Parsing text fucking sucks. Even more so when it's wiki text with no consistencies whatsoever
-    # This is wildly inefficent but I don't. fucking. care. Just make it work
-    text = regex.sub(
-        r"(?=\{)(\{([^{}]|(?1))*\})|\[\[(Kategori|Fil):.+?\]\]|\{.+?\}", "", text, flags=regex.MULTILINE | regex.DOTALL
-    )  # Remove templates, categories and images
-    text = pypandoc.convert_text(text, "markdown", format="mediawiki")
-    text = regex.sub(
-        r"\[(.+?)\]\(.+? \"wikilink\"\)", r"\1", text, flags=regex.MULTILINE | regex.DOTALL
-    )  # Pandoc adds alt text to links. discord doesn't support that
-    text = regex.sub(r"\{.+\}", "", text)  # Pandoc adds section links. Remove them
-    text = text.strip("\n")
-    text = text[:1000] + "..." if len(text) > 1000 else text
 
-    return title, url, text, image
+
+async def fetch_page_image(session: aiohttp.ClientSession, title: str) -> str | None:
+    """
+    Fetch the main image of an article from Viteboka
+
+    Parameters
+    ----------
+    session (aiohttp.ClientSession): The HTTP session to use
+    title (str): The title of the article
+
+    Returns
+    ----------
+    str | None: The url of the image. None if the article has no image or it couldn't be fetched
+    """
+
+    params = {
+        "action": "query",
+        "format": "json",
+        "formatversion": 2,
+        "titles": title,
+        "prop": "pageimages",
+        "piprop": "original",
+    }
+
+    async with session.get(API_URL, params=params) as response:
+        if response.status != 200:
+            return None
+        pages = (await response.json(content_type=None))["query"]["pages"]
+
+    return pages[0].get("original", {}).get("source")
+
+
+def shorten(text: str, limit: int) -> str:
+    """
+    Shorten markdown text to fit within a limit.
+    Pandoc puts every paragraph and list item on its own line, so whole lines are kept where possible
+
+    Parameters
+    ----------
+    text (str): The markdown text
+    limit (int): The maximum length
+
+    Returns
+    ----------
+    str: The shortened text
+    """
+
+    if len(text) <= limit:
+        return text
+
+    shortened = ""
+    for line in text.split("\n"):
+        if len(shortened) + len(line) + len("\n...") > limit:
+            break
+        shortened += line + "\n"
+
+    room = limit - len(shortened) - len("...")
+    if room < 100 and shortened.strip():
+        return shortened.rstrip() + "\n..."
+
+    # Fill the rest with the start of the line that didn't fit. Its links are removed so cutting it can't break one
+    line = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", line)
+    if len(line) <= room:
+        return shortened + line + "\n..."
+    return shortened + line[:room].rsplit(" ", 1)[0] + "..."
 
 
 def viteboka_embed(title: str, url: str, text: str, image: str):

uv.lock

@@ -1011,7 +1011,6 @@ dependencies = [
     { name = "psycopg-pool" },
     { name = "pypandoc" },
     { name = "pyyaml" },
-    { name = "regex" },
     { name = "tzdata" },
     { name = "wordcloud" },
     { name = "yt-dlp" },
@@ -1040,7 +1039,6 @@ requires-dist = [
     { name = "psycopg-pool", specifier = "==3.3.*" },
     { name = "pypandoc", specifier = "==1.17.*" },
     { name = "pyyaml", specifier = "==6.0.*" },
-    { name = "regex", specifier = "==2026.9.29" },
     { name = "tzdata" },
     { name = "wordcloud", specifier = "==1.9.*" },
     { name = "yt-dlp", specifier = "==2026.10.21" },