dnscms: remove some wordpress import functionality, keep fields around but defer

dnscms/associations/models.py

@@ -9,12 +9,16 @@ from grapple.models import (
 )
 from wagtail.admin.panels import FieldPanel
 from wagtail.fields import RichTextField
-from wagtail.models import Page
+from wagtail.models import Page, PageManager
 from wagtail.search import index
 from wagtail_headless_preview.models import HeadlessMixin
 
 from dnscms.fields import CommonStreamField
-from dnscms.wordpress.models import WPImportedPageMixin
+from dnscms.wordpress.models import DeferWPFieldsManagerMixin, WPImportedPageMixin
+
+
+class AssociationPageManager(DeferWPFieldsManagerMixin, PageManager):
+    pass
 
 
 @register_singular_query_field("associationIndex")
@@ -48,6 +52,8 @@ class AssociationPage(HeadlessMixin, WPImportedPageMixin, Page):
     parent_page_types = ["associations.AssociationIndex"]
     show_in_menus = False
 
+    objects = AssociationPageManager()
+
     class AssociationType(models.TextChoices):
         FORENING = "forening", _("Association")
         UTVALG = "utvalg", _("Committee")
@@ -97,33 +103,3 @@ class AssociationPage(HeadlessMixin, WPImportedPageMixin, Page):
     class Meta:
         verbose_name = _("association")
         verbose_name_plural = _("associations")
-
-    def import_wordpress_data(self, data):
-        import html
-
-        # Wagtail page model fields
-        self.title = html.unescape(data["title"])
-        self.slug = data["slug"]
-        self.first_published_at = data["first_published_at"]
-        self.last_published_at = data["last_published_at"]
-        self.latest_revision_created_at = data["latest_revision_created_at"]
-        self.search_description = data["search_description"]
-
-        # debug fields
-        self.wp_post_id = data["wp_post_id"]
-        self.wp_post_type = data["wp_post_type"]
-        self.wp_link = data["wp_link"]
-        self.wp_raw_content = data["wp_raw_content"]
-        self.wp_block_json = data["wp_block_json"]
-        self.wp_processed_content = data["wp_processed_content"]
-        self.wp_normalized_styles = data["wp_normalized_styles"]
-        self.wp_post_meta = data["wp_post_meta"]
-
-        # own model fields
-        self.body = data["body"] or ""
-
-        meta = data["wp_post_meta"]
-        self.association_type = meta.get("neuf_associations_type").lower()
-        self.website_url = meta.get("neuf_associations_homepage") or ""
-
-        self.excerpt = meta.get("excerpt_encoded") or "TODO"

dnscms/dnscms/settings/base.py

@@ -12,7 +12,6 @@ https://docs.djangoproject.com/en/4.2/ref/settings/
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 import os
-import re
 
 PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 BASE_DIR = os.path.dirname(PROJECT_DIR)
@@ -228,74 +227,3 @@ GRAPPLE = {
     "PAGE_SIZE": 100,
     "MAX_PAGE_SIZE": 5000,
 }
-
-# Wgtail WordPress import
-WAGTAIL_WORDPRESS_IMPORTER_SOURCE_DOMAIN = "https://studentersamfundet.no/"
-WAGTAIL_WORDPRESS_IMPORTER_CONVERT_HTML_TAGS_TO_BLOCKS = {
-    # "h1": "wagtail_wordpress_import.block_builder_defaults.build_heading_block",
-    "table": "wagtail_wordpress_import.block_builder_defaults.build_table_block",
-    # "iframe": "wagtail_wordpress_import.block_builder_defaults.build_iframe_block",
-    # "form": "wagtail_wordpress_import.block_builder_defaults.build_form_block",
-    # "img": "wagtail_wordpress_import.block_builder_defaults.build_image_block",
-    # "blockquote": "wagtail_wordpress_import.block_builder_defaults.build_block_quote_block",
-}
-WAGTAIL_WORDPRESS_IMPORTER_FALLBACK_BLOCK = (
-    "dnscms.wordpress.block_builder.build_richtext_block_content"
-)
-WORDPRESS_IMPORT_HOOKS_ITEMS_TO_CACHE = {
-    "attachment": {
-        "DATA_TAG": "thumbnail_id",
-        "FUNCTION": "dnscms.wordpress.import_hooks.header_image_processor",
-    }
-}
-# WORDPRESS_IMPORT_HOOKS_TAGS_TO_CACHE = {
-#     "wp:term": {
-#         "DATA_TAG": "category",
-#         "FUNCTION": "dnscms.wordpress.import_hooks.categories_processor",
-#     }
-# }
-WAGTAIL_WORDPRESS_IMPORTER_PROMOTE_CHILD_TAGS = {
-    "TAGS_TO_PROMOTE": [],
-    "PARENTS_TO_REMOVE": ["p", "div", "span"],
-}
-WAGTAIL_WORDPRESS_IMPORT_PREFILTERS = [
-    {
-        "FUNCTION": "wagtail_wordpress_import.prefilters.linebreaks_wp",
-    },
-    {
-        "FUNCTION": "wagtail_wordpress_import.prefilters.transform_shortcodes",
-    },
-    {
-        "FUNCTION": "wagtail_wordpress_import.prefilters.transform_inline_styles",
-        "OPTIONS": {
-            "TRANSFORM_STYLES_MAPPING": [
-                (
-                    re.compile(r"font-style:italic;font-weight:bold;", re.IGNORECASE),
-                    "wagtail_wordpress_import.prefilters.transform_styles_defaults.transform_style_bold_italic",
-                ),
-                (
-                    re.compile(r"font-weight:bold;", re.IGNORECASE),
-                    "wagtail_wordpress_import.prefilters.transform_styles_defaults.transform_style_bold",
-                ),
-                (
-                    re.compile(r"font-style:italic;", re.IGNORECASE),
-                    "wagtail_wordpress_import.prefilters.transform_styles_defaults.transform_style_italic",
-                ),
-                # (
-                #     re.compile(
-                #         r"text-align:center;",
-                #         re.IGNORECASE,
-                #     ),
-                #     transform_style_center,
-                # ),
-                # (re.compile(r"text-align:left;", re.IGNORECASE), transform_style_left),
-                # (re.compile(r"text-align:right;", re.IGNORECASE), transform_style_right),
-                # (re.compile(r"float:left;", re.IGNORECASE), transform_float_left),
-                # (re.compile(r"float:right;", re.IGNORECASE), transform_float_right),
-            ],
-        },
-    },
-    {
-        "FUNCTION": "wagtail_wordpress_import.prefilters.bleach_clean",
-    },
-]

dnscms/dnscms/wordpress/block_builder.py

@@ -1,24 +0,0 @@
-from django.conf import settings
-from wagtail_wordpress_import.block_builder_defaults import (
-    document_linker,
-    image_linker,
-    import_string,
-)
-
-
-def build_richtext_block_content(html, blocks):
-    """
-    image_linker is called to link up and retrive the remote image
-    document_linker is called to link up and retrive the remote documents
-    filters are called to replace inline shortcodes
-    """
-    html = image_linker(html)
-    html = document_linker(html)
-    for inline_shortcode_handler in getattr(
-        settings, "WAGTAIL_WORDPRESS_IMPORTER_INLINE_SHORTCODE_HANDLERS", []
-    ):
-        function = import_string(inline_shortcode_handler).construct_html_tag
-        html = function(html)
-    blocks.append({"type": "paragraph", "value": html})
-    html = ""
-    return html

dnscms/dnscms/wordpress/import_hooks.py

@@ -1,51 +0,0 @@
-from wagtail_wordpress_import.block_builder_defaults import get_or_save_image
-
-
-def header_image_processor(imported_pages, data_tag, items_cache):
-    """
-    imported_pages:
-        Is a specific() page model queryset of all imported pages.
-    data_tag:
-        Is the value of the `DATA_TAG` key from the configuration above.
-    items_cache:
-        Is a list of dictionaries, one for each item in the XML file.
-    """
-
-    # See note above about leading _ and : characters in the XML value
-    lookup = f"wp_post_meta__{data_tag}"
-
-    for attachment in items_cache:
-        # The id of the cached item used in the filter
-        thumbnail_id = attachment.get("wp:post_id")
-
-        # Filter the imported_pages for only pages that include the
-        # matching thumbnail_id in the wp_post_meta field
-        pages = imported_pages.filter(**{lookup: thumbnail_id})
-
-        if pages.exists():
-            # guid is the url of the image to fetch, the get_or_save_image()
-            # function will fetch the image if it doesn't exist
-            image_url = attachment.get("guid")
-            # fix cases where the /wp prefix is missing from the image url
-            if image_url.startswith("https://studentersamfundet.no/wp-content/uploads/"):
-                image_url = image_url.replace(
-                    "https://studentersamfundet.no/wp-content/uploads/",
-                    "https://studentersamfundet.no/wp/wp-content/uploads/",
-                )
-            try:
-                image = get_or_save_image(image_url)
-            except Exception as e:
-                print("Error with image", image_url, "associated with pages:", pages)
-                print(e)
-                continue
-
-            print("Attaching header images to pages:", pages)
-            try:
-                pages.update(featured_image=image)
-            except Exception:
-                pass
-
-            try:
-                pages.update(logo=image)
-            except Exception:
-                pass

dnscms/dnscms/wordpress/models.py

@@ -1,17 +1,84 @@
+from django.core.exceptions import FieldDoesNotExist
 from django.db import models
 from wagtail.models import Page
 
+# Field names declared by WPImportedPageMixin. Concrete models that mix it in
+# get a manager that .defer()s these so they're never loaded by default — the
+# columns stay in the database (no migration), and any code path that
+# explicitly reads them still works via Django's lazy-load.
+WP_IMPORT_FIELDS = (
+    "wp_post_id",
+    "wp_post_type",
+    "wp_link",
+    "wp_raw_content",
+    "wp_processed_content",
+    "wp_block_json",
+    "wp_normalized_styles",
+    "wp_post_meta",
+)
+
 
 # https://github.com/wagtail/wagtail-wordpress-import/blob/main/wagtail_wordpress_import/models.py
+# DJ001 (null=True on string fields) is suppressed: the schema mirrors the
+# upstream mixin and changing nullability would force a migration we avoid.
 class WPImportedPageMixin(Page):
     wp_post_id = models.IntegerField(blank=True, null=True)
-    wp_post_type = models.CharField(max_length=255, blank=True, null=True)
+    wp_post_type = models.CharField(max_length=255, blank=True, null=True)  # noqa: DJ001
-    wp_link = models.TextField(blank=True, null=True)
+    wp_link = models.TextField(blank=True, null=True)  # noqa: DJ001
-    wp_raw_content = models.TextField(blank=True, null=True)
+    wp_raw_content = models.TextField(blank=True, null=True)  # noqa: DJ001
-    wp_processed_content = models.TextField(blank=True, null=True)
+    wp_processed_content = models.TextField(blank=True, null=True)  # noqa: DJ001
-    wp_block_json = models.TextField(blank=True, null=True)
+    wp_block_json = models.TextField(blank=True, null=True)  # noqa: DJ001
-    wp_normalized_styles = models.TextField(blank=True, null=True)
+    wp_normalized_styles = models.TextField(blank=True, null=True)  # noqa: DJ001
     wp_post_meta = models.JSONField(blank=True, null=True)
 
     class Meta:
         abstract = True
+
+
+class DeferWPFieldsManagerMixin:
+    """
+    Manager mixin that always .defer()s the wp_* import columns, so they are
+    never SELECTed by default queries. The columns remain in the database;
+    accessing one on an instance still works (Django lazy-loads it).
+    """
+
+    def get_queryset(self):
+        return super().get_queryset().defer(*WP_IMPORT_FIELDS)
+
+
+def _resolve_related_model(model, lookup_path):
+    """Walk a ``foo__bar`` lookup path and return the final related model, or None."""
+    current = model
+    for part in lookup_path.split("__"):
+        try:
+            field = current._meta.get_field(part)
+        except FieldDoesNotExist:
+            return None
+        current = getattr(field, "related_model", None)
+        if current is None:
+            return None
+    return current
+
+
+class WPAwareQuerySet(models.QuerySet):
+    """
+    QuerySet whose ``select_related()`` auto-defers wp_* columns when a join
+    targets a WPImportedPageMixin model. Without this, ``select_related``
+    builds a JOIN that ignores the related model's manager and SELECTs every
+    column including the wp_* blobs. Apply via ``objects = WPAwareManager()``
+    on any model that has a ForeignKey into a WPImported page.
+    """
+
+    def select_related(self, *fields):
+        qs = super().select_related(*fields)
+        defers = [
+            f"{path}__{name}"
+            for path in fields
+            if (related := _resolve_related_model(qs.model, path)) is not None
+            and issubclass(related, WPImportedPageMixin)
+            for name in WP_IMPORT_FIELDS
+        ]
+        return qs.defer(*defers) if defers else qs
+
+
+WPAwareManager = models.Manager.from_queryset(WPAwareQuerySet)

dnscms/events/migrations/0055_alter_eventcategory_options_and_more.py

@@ -0,0 +1,82 @@
+# Generated by Django 6.0.5 on 2026-05-22 23:27
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('associations', '0026_alter_association_options'),
+        ('events', '0054_alter_eventpage_options'),
+        ('images', '0005_customimage_description'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='eventcategory',
+            options={'ordering': ['name'], 'verbose_name': 'event category', 'verbose_name_plural': 'event categories'},
+        ),
+        migrations.AlterModelOptions(
+            name='eventoccurrence',
+            options={'verbose_name': 'occurrence', 'verbose_name_plural': 'occurrences'},
+        ),
+        migrations.AlterModelOptions(
+            name='eventorganizer',
+            options={'ordering': ['name'], 'verbose_name': 'event organizer', 'verbose_name_plural': 'event organizers'},
+        ),
+        migrations.AlterModelOptions(
+            name='eventorganizerlink',
+            options={'verbose_name': 'organizer', 'verbose_name_plural': 'organizers'},
+        ),
+        migrations.AlterField(
+            model_name='eventcategory',
+            name='pig',
+            field=models.CharField(blank=True, choices=[('', 'None'), ('logo', 'Logogrisen'), ('music', 'Musikergrisen'), ('drink', 'Drikkegrisen'), ('dance', 'Dansegrisen'), ('point', 'Pekegrisen'), ('student', 'Studentgrisen'), ('listen', 'Lyttegrisen'), ('guard', 'Vaktgrisen'), ('key', 'Nøkkelgrisen'), ('chill', 'Liggegrisen'), ('peek', 'Tittegrisen')], default='', help_text='Default pig for events of this kind.', max_length=32),
+        ),
+        migrations.AlterField(
+            model_name='eventcategory',
+            name='show_in_filters',
+            field=models.BooleanField(default=False, help_text='Should this category be available as a filter in the event programme?'),
+        ),
+        migrations.AlterField(
+            model_name='eventoccurrence',
+            name='venue_custom',
+            field=models.CharField(blank=True, help_text='Use this <em>if none of the venues that can be selected on the left</em> fit. E.g. <em>Frederikkeplassen</em> or <em>Sirkusteltet</em>.', max_length=128),
+        ),
+        migrations.AlterField(
+            model_name='eventorganizer',
+            name='association',
+            field=models.ForeignKey(blank=True, help_text='If a DNS association or committee is behind it, choose it here.', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='organizers', to='associations.associationpage'),
+        ),
+        migrations.AlterField(
+            model_name='eventorganizer',
+            name='external_url',
+            field=models.URLField(blank=True, help_text="Link to the external organizer's website", max_length=512),
+        ),
+        migrations.AlterField(
+            model_name='eventpage',
+            name='facebook_url',
+            field=models.URLField(blank=True, help_text='Direct link to the event on Facebook', max_length=1024),
+        ),
+        migrations.AlterField(
+            model_name='eventpage',
+            name='featured_image',
+            field=models.ForeignKey(blank=True, help_text="Choose an image for use in the programme and other surfaces. Should be a photo or an illustration without too much text – don't reuse a Facebook cover uncritically!", null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='images.customimage'),
+        ),
+        migrations.AlterField(
+            model_name='eventpage',
+            name='pig',
+            field=models.CharField(blank=True, choices=[('', 'None'), ('automatic', 'Automatic'), ('logo', 'Logogrisen'), ('music', 'Musikergrisen'), ('drink', 'Drikkegrisen'), ('dance', 'Dansegrisen'), ('point', 'Pekegrisen'), ('student', 'Studentgrisen'), ('listen', 'Lyttegrisen'), ('guard', 'Vaktgrisen'), ('key', 'Nøkkelgrisen'), ('chill', 'Liggegrisen'), ('peek', 'Tittegrisen')], default='automatic', help_text="The pig that hangs out on the event page. Automatic causes one to be chosen based on the event's category.", max_length=32),
+        ),
+        migrations.AlterField(
+            model_name='eventpage',
+            name='subtitle',
+            field=models.CharField(blank=True, help_text='A short text that appears right below the title. Feel free to leave it empty if you fit most of it in the main title.', max_length=128),
+        ),
+        migrations.AlterField(
+            model_name='eventpage',
+            name='ticket_url',
+            field=models.URLField(blank=True, help_text='Direct link to ticket purchase, e.g. TicketCo, Billetto or Ticketmaster', max_length=1024),
+        ),
+    ]

dnscms/events/models.py

@@ -35,7 +35,11 @@ from wagtail_headless_preview.models import HeadlessMixin
 from associations.widgets import AssociationChooserWidget
 from dnscms.fields import CommonStreamField
 from dnscms.options import ALL_PIGS
-from dnscms.wordpress.models import WPImportedPageMixin
+from dnscms.wordpress.models import (
+    DeferWPFieldsManagerMixin,
+    WPAwareManager,
+    WPImportedPageMixin,
+)
 from venues.models import VenuePage
 
 
@@ -163,6 +167,8 @@ class EventOrganizerLink(Orderable):
 @register_snippet
 @register_query_field("eventOrganizer", "eventOrganizers")
 class EventOrganizer(ClusterableModel):
+    objects = WPAwareManager()
+
     name = models.CharField(
         max_length=100,
         null=False,
@@ -235,7 +241,11 @@ class EventPageQuerySet(PageQuerySet):
         )
 
 
-EventPageManager = PageManager.from_queryset(EventPageQuerySet)
+class EventPageManager(
+    DeferWPFieldsManagerMixin,
+    PageManager.from_queryset(EventPageQuerySet),
+):
+    pass
 
 
 class EventPage(HeadlessMixin, WPImportedPageMixin, Page):
@@ -434,130 +444,10 @@ class EventPage(HeadlessMixin, WPImportedPageMixin, Page):
             self.price_student = ""
             self.price_member = ""
 
-    def import_wordpress_data(self, data):
-        import datetime
-        import html
-        from zoneinfo import ZoneInfo
-
-        from django.core.validators import URLValidator
-
-        validate_url = URLValidator(schemes=["http", "https"])
-
-        def fix_url(url):
-            if not url:
-                return None
-            url = url.strip()
-            try:
-                validate_url(url)
-            except Exception:
-                print(f"Bogus URL for {self.wp_post_id}: {url}")
-                return None
-            return url
-
-        # Wagtail page model fields
-        self.title = html.unescape(data["title"])
-        self.slug = data["slug"]
-        self.first_published_at = data["first_published_at"]
-        self.last_published_at = data["last_published_at"]
-        self.latest_revision_created_at = data["latest_revision_created_at"]
-        self.search_description = data["search_description"]
-
-        # debug fields
-        self.wp_post_id = data["wp_post_id"]
-        self.wp_post_type = data["wp_post_type"]
-        self.wp_link = data["wp_link"]
-        self.wp_raw_content = data["wp_raw_content"]
-        self.wp_block_json = data["wp_block_json"]
-        self.wp_processed_content = data["wp_processed_content"]
-        self.wp_normalized_styles = data["wp_normalized_styles"]
-        self.wp_post_meta = data["wp_post_meta"]
-
-        # own model fields
-        self.body = data["body"] or ""
-
-        # categories (organizers and event types)
-        wp_categories = data["wp_categories"]
-
-        # organizers
-        organizer_cats = [x for x in wp_categories if x["domain"] == "event_organizer"]
-        organizers = []
-        for x in organizer_cats:
-            try:
-                organizer = EventOrganizer.objects.get(slug=x["nicename"])
-            except EventOrganizer.DoesNotExist:
-                organizer = EventOrganizer.objects.create(name=x["name"], slug=x["nicename"])
-            organizers.append(organizer)
-
-        self.organizer_links.set(
-            [EventOrganizerLink(event=self, organizer=organizer) for organizer in organizers]
-        )
-
-        ## event types
-        # type_cats = [x for x in wp_categories if x["domain"] == "event_type"]
-        # event_categories = []
-        # for x in type_cats:
-        #     try:
-        #         event_category = EventCategory.objects.get(slug=x["nicename"])
-        #     except EventCategory.DoesNotExist:
-        #         event_category = EventCategory.objects.create(
-        #             name=x["name"], slug=x["nicename"], show_in_filters=False
-        #         )
-        #     event_categories.append(event_category)
-        # self.categories.set(event_categories)
-
-        meta = data["wp_post_meta"]
-
-        start_ts = meta.get("neuf_events_starttime") or 1337
-        end_ts = meta.get("neuf_events_endtime")
-        tz = ZoneInfo("Europe/Oslo")
-        start = start_ts and datetime.datetime.fromtimestamp(start_ts, datetime.UTC).replace(
-            tzinfo=tz
-        )
-        end = end_ts and datetime.datetime.fromtimestamp(end_ts, datetime.UTC).replace(tzinfo=tz)
-        venue_id = meta.get("neuf_events_venue_id")
-        venue_custom = meta.get("neuf_events_venue")
-        venue = None
-        if venue_id:
-            venue = VenuePage.objects.get(wp_post_id=venue_id)
-            venue_custom = ""
-        else:
-            venue_custom = venue_custom or ""
-
-        occurrence = EventOccurrence(
-            event=self, start=start, end=end, venue=venue, venue_custom=venue_custom
-        )
-        self.occurrences.set([occurrence])
-
-        self.ticket_url = fix_url(meta.get("neuf_events_bs_url")) or ""
-        self.facebook_url = fix_url(meta.get("neuf_events_fb_url")) or ""
-
-        def parse_price(price):
-            if price is None:
-                return ""
-            if type(price) is int:
-                return price
-            p = price.strip()
-            if p == "":
-                return ""
-            try:
-                return int(p)
-            except ValueError:
-                pass
-            free = ["gratis", "free", "gratis/free", "free/gratis"]
-            if p.lower() in free:
-                return 0
-            return price
-
-        price_regular = parse_price(meta.get("neuf_events_price_regular"))
-        price_member = parse_price(meta.get("neuf_events_price_member"))
-        if not price_regular and not price_member:
-            self.free = True
-        else:
-            self.price_regular = parse_price(meta.get("neuf_events_price_regular"))
-            self.price_member = parse_price(meta.get("neuf_events_price_member"))
-
 
 class EventOccurrence(Orderable):
+    objects = WPAwareManager()
+
     event = ParentalKey(EventPage, on_delete=models.CASCADE, related_name="occurrences")
     start = models.DateTimeField()
     end = models.DateTimeField(null=True, blank=True)

dnscms/news/models.py

@@ -4,12 +4,16 @@ from grapple.helpers import register_singular_query_field
 from grapple.models import GraphQLImage, GraphQLRichText, GraphQLStreamfield, GraphQLString
 from wagtail.admin.panels import FieldPanel
 from wagtail.fields import RichTextField
-from wagtail.models import Page
+from wagtail.models import Page, PageManager
 from wagtail.search import index
 from wagtail_headless_preview.models import HeadlessMixin
 
 from dnscms.fields import CommonStreamField
-from dnscms.wordpress.models import WPImportedPageMixin
+from dnscms.wordpress.models import DeferWPFieldsManagerMixin, WPImportedPageMixin
+
+
+class NewsPageManager(DeferWPFieldsManagerMixin, PageManager):
+    pass
 
 
 @register_singular_query_field("newsIndex")
@@ -39,6 +43,8 @@ class NewsPage(HeadlessMixin, WPImportedPageMixin, Page):
     parent_page_types = ["news.NewsIndex"]
     show_in_menus = False
 
+    objects = NewsPageManager()
+
     excerpt = models.TextField(max_length=512, blank=False)
     lead = RichTextField(features=["italic", "link"], blank=True)
     body = CommonStreamField
@@ -90,52 +96,3 @@ class NewsPage(HeadlessMixin, WPImportedPageMixin, Page):
     class Meta:
         verbose_name = _("news article")
         verbose_name_plural = _("news articles")
-
-    def import_wordpress_data(self, data):
-        import html
-
-        from bs4 import BeautifulSoup
-
-        def generate_excerpt(html_content):
-            soup = BeautifulSoup(html_content, features="lxml")
-            VALID_TAGS = ["div", "p"]
-
-            for tag in soup.findAll("p"):
-                if tag.name not in VALID_TAGS:
-                    tag.remove()
-
-            text = soup.get_text().strip()
-            words = text.split(" ")
-            if len(words) < 26:
-                return text
-            return " ".join(words[:25]) + " [...]"
-
-        # Wagtail page model fields
-        self.title = html.unescape(data["title"])
-        self.slug = data["slug"]
-        self.first_published_at = data["first_published_at"]
-        self.last_published_at = data["last_published_at"]
-        self.latest_revision_created_at = data["latest_revision_created_at"]
-        self.search_description = data["search_description"]
-
-        # debug fields
-        self.wp_post_id = data["wp_post_id"]
-        self.wp_post_type = data["wp_post_type"]
-        self.wp_link = data["wp_link"]
-        self.wp_raw_content = data["wp_raw_content"]
-        self.wp_block_json = data["wp_block_json"]
-        self.wp_processed_content = data["wp_processed_content"]
-        self.wp_normalized_styles = data["wp_normalized_styles"]
-        self.wp_post_meta = data["wp_post_meta"]
-
-        # own model fields
-        self.body = data["body"] or ""
-
-        meta = data["wp_post_meta"]
-
-        written_excerpt = meta.get("excerpt_encoded")
-        generated_excerpt = ""
-        if not written_excerpt:
-            generated_excerpt = generate_excerpt(self.wp_processed_content)
-
-        self.excerpt = written_excerpt or generated_excerpt or "[...]"

dnscms/tests/test_events.py

@@ -248,6 +248,22 @@ def test_future_events_does_not_have_n_plus_one_queries(
     )
 
 
+def test_future_events_does_not_load_wp_import_fields(event_index, graphql_post):
+    """wp_* columns must stay deferred and lazy-load on explicit access."""
+    event = EventPageFactory(parent=event_index, wp_raw_content="marker")
+    EventOccurrence.objects.create(
+        event=event, start=timezone.now() + timedelta(days=1), venue_custom="X"
+    )
+
+    with CaptureQueriesContext(connection) as ctx:
+        response, body = graphql_post("{ eventIndex { futureEvents { id } } }")
+
+    assert response.status_code == 200 and "errors" not in body, body
+    sql = "\n".join(q["sql"] for q in ctx.captured_queries)
+    assert "wp_raw_content" not in sql, f"wp_* must be deferred. SQL:\n{sql}"
+    assert EventPage.objects.get(pk=event.pk).wp_raw_content == "marker"
+
+
 def test_graphql_event_index_future_events_ordered_by_next_occurrence(event_index, graphql_post):
     now = timezone.now()
 

dnscms/venues/models.py

@@ -9,13 +9,17 @@ from grapple.models import (
 )
 from wagtail.admin.panels import FieldPanel, FieldRowPanel, MultiFieldPanel
 from wagtail.fields import RichTextField, StreamField
-from wagtail.models import Page
+from wagtail.models import Page, PageManager
 from wagtail.search import index
 from wagtail_headless_preview.models import HeadlessMixin
 
 from dnscms.blocks import ImageSliderBlock
 from dnscms.fields import CommonStreamField
-from dnscms.wordpress.models import WPImportedPageMixin
+from dnscms.wordpress.models import DeferWPFieldsManagerMixin, WPImportedPageMixin
+
+
+class VenuePageManager(DeferWPFieldsManagerMixin, PageManager):
+    pass
 
 
 @register_singular_query_field("venueIndex")
@@ -59,6 +63,8 @@ class VenuePage(HeadlessMixin, WPImportedPageMixin, Page):
     # should not be able to be shown in menus
     show_in_menus = False
 
+    objects = VenuePageManager()
+
     featured_image = models.ForeignKey(
         "images.CustomImage",
         null=True,
@@ -178,42 +184,3 @@ class VenuePage(HeadlessMixin, WPImportedPageMixin, Page):
     search_fields = Page.search_fields + [
         index.SearchField("body"),
     ]
-
-    def import_wordpress_data(self, data):
-        import html
-
-        # Wagtail page model fields
-        self.title = html.unescape(data["title"])
-        self.slug = data["slug"]
-        self.first_published_at = data["first_published_at"]
-        self.last_published_at = data["last_published_at"]
-        self.latest_revision_created_at = data["latest_revision_created_at"]
-        self.search_description = data["search_description"]
-
-        # debug fields
-        self.wp_post_id = data["wp_post_id"]
-        self.wp_post_type = data["wp_post_type"]
-        self.wp_link = data["wp_link"]
-        self.wp_raw_content = data["wp_raw_content"]
-        self.wp_block_json = data["wp_block_json"]
-        self.wp_processed_content = data["wp_processed_content"]
-        self.wp_normalized_styles = data["wp_normalized_styles"]
-        self.wp_post_meta = data["wp_post_meta"]
-
-        # own model fields
-        self.body = data["body"] or ""
-
-        meta = data["wp_post_meta"]
-        self.show_as_bookable = meta.get("neuf_venues_show_on_booking_page", False)
-        self.preposition = meta.get("neuf_venues_preposition") or ""
-        self.floor = meta.get("neuf_venues_floor") or ""
-        self.used_for = meta.get("neuf_venues_used_for") or ""
-
-        self.capability_bar = meta.get("neuf_venues_bar") or ""
-        self.capability_audio = meta.get("neuf_venues_audio") or ""
-        self.capability_lighting = meta.get("neuf_venues_lighting") or ""
-        self.capability_audio_video = meta.get("neuf_venues_audio_video") or ""
-
-        self.capacity_legal = meta.get("neuf_venues_capacity_legal") or ""
-        self.capacity_standing = meta.get("neuf_venues_capacity_standing") or ""
-        self.capacity_sitting = meta.get("neuf_venues_capacity_sitting") or ""