dnscms: remove some wordpress import functionality, keep fields around but defer
This commit is contained in:
@@ -12,7 +12,6 @@ https://docs.djangoproject.com/en/4.2/ref/settings/
|
||||
|
||||
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
|
||||
import os
|
||||
import re
|
||||
|
||||
PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
BASE_DIR = os.path.dirname(PROJECT_DIR)
|
||||
@@ -228,74 +227,3 @@ GRAPPLE = {
|
||||
"PAGE_SIZE": 100,
|
||||
"MAX_PAGE_SIZE": 5000,
|
||||
}
|
||||
|
||||
# Wgtail WordPress import
|
||||
WAGTAIL_WORDPRESS_IMPORTER_SOURCE_DOMAIN = "https://studentersamfundet.no/"
|
||||
WAGTAIL_WORDPRESS_IMPORTER_CONVERT_HTML_TAGS_TO_BLOCKS = {
|
||||
# "h1": "wagtail_wordpress_import.block_builder_defaults.build_heading_block",
|
||||
"table": "wagtail_wordpress_import.block_builder_defaults.build_table_block",
|
||||
# "iframe": "wagtail_wordpress_import.block_builder_defaults.build_iframe_block",
|
||||
# "form": "wagtail_wordpress_import.block_builder_defaults.build_form_block",
|
||||
# "img": "wagtail_wordpress_import.block_builder_defaults.build_image_block",
|
||||
# "blockquote": "wagtail_wordpress_import.block_builder_defaults.build_block_quote_block",
|
||||
}
|
||||
WAGTAIL_WORDPRESS_IMPORTER_FALLBACK_BLOCK = (
|
||||
"dnscms.wordpress.block_builder.build_richtext_block_content"
|
||||
)
|
||||
WORDPRESS_IMPORT_HOOKS_ITEMS_TO_CACHE = {
|
||||
"attachment": {
|
||||
"DATA_TAG": "thumbnail_id",
|
||||
"FUNCTION": "dnscms.wordpress.import_hooks.header_image_processor",
|
||||
}
|
||||
}
|
||||
# WORDPRESS_IMPORT_HOOKS_TAGS_TO_CACHE = {
|
||||
# "wp:term": {
|
||||
# "DATA_TAG": "category",
|
||||
# "FUNCTION": "dnscms.wordpress.import_hooks.categories_processor",
|
||||
# }
|
||||
# }
|
||||
WAGTAIL_WORDPRESS_IMPORTER_PROMOTE_CHILD_TAGS = {
|
||||
"TAGS_TO_PROMOTE": [],
|
||||
"PARENTS_TO_REMOVE": ["p", "div", "span"],
|
||||
}
|
||||
WAGTAIL_WORDPRESS_IMPORT_PREFILTERS = [
|
||||
{
|
||||
"FUNCTION": "wagtail_wordpress_import.prefilters.linebreaks_wp",
|
||||
},
|
||||
{
|
||||
"FUNCTION": "wagtail_wordpress_import.prefilters.transform_shortcodes",
|
||||
},
|
||||
{
|
||||
"FUNCTION": "wagtail_wordpress_import.prefilters.transform_inline_styles",
|
||||
"OPTIONS": {
|
||||
"TRANSFORM_STYLES_MAPPING": [
|
||||
(
|
||||
re.compile(r"font-style:italic;font-weight:bold;", re.IGNORECASE),
|
||||
"wagtail_wordpress_import.prefilters.transform_styles_defaults.transform_style_bold_italic",
|
||||
),
|
||||
(
|
||||
re.compile(r"font-weight:bold;", re.IGNORECASE),
|
||||
"wagtail_wordpress_import.prefilters.transform_styles_defaults.transform_style_bold",
|
||||
),
|
||||
(
|
||||
re.compile(r"font-style:italic;", re.IGNORECASE),
|
||||
"wagtail_wordpress_import.prefilters.transform_styles_defaults.transform_style_italic",
|
||||
),
|
||||
# (
|
||||
# re.compile(
|
||||
# r"text-align:center;",
|
||||
# re.IGNORECASE,
|
||||
# ),
|
||||
# transform_style_center,
|
||||
# ),
|
||||
# (re.compile(r"text-align:left;", re.IGNORECASE), transform_style_left),
|
||||
# (re.compile(r"text-align:right;", re.IGNORECASE), transform_style_right),
|
||||
# (re.compile(r"float:left;", re.IGNORECASE), transform_float_left),
|
||||
# (re.compile(r"float:right;", re.IGNORECASE), transform_float_right),
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"FUNCTION": "wagtail_wordpress_import.prefilters.bleach_clean",
|
||||
},
|
||||
]
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
from django.conf import settings
|
||||
from wagtail_wordpress_import.block_builder_defaults import (
|
||||
document_linker,
|
||||
image_linker,
|
||||
import_string,
|
||||
)
|
||||
|
||||
|
||||
def build_richtext_block_content(html, blocks):
|
||||
"""
|
||||
image_linker is called to link up and retrive the remote image
|
||||
document_linker is called to link up and retrive the remote documents
|
||||
filters are called to replace inline shortcodes
|
||||
"""
|
||||
html = image_linker(html)
|
||||
html = document_linker(html)
|
||||
for inline_shortcode_handler in getattr(
|
||||
settings, "WAGTAIL_WORDPRESS_IMPORTER_INLINE_SHORTCODE_HANDLERS", []
|
||||
):
|
||||
function = import_string(inline_shortcode_handler).construct_html_tag
|
||||
html = function(html)
|
||||
blocks.append({"type": "paragraph", "value": html})
|
||||
html = ""
|
||||
return html
|
||||
@@ -1,51 +0,0 @@
|
||||
from wagtail_wordpress_import.block_builder_defaults import get_or_save_image
|
||||
|
||||
|
||||
def header_image_processor(imported_pages, data_tag, items_cache):
|
||||
"""
|
||||
imported_pages:
|
||||
Is a specific() page model queryset of all imported pages.
|
||||
data_tag:
|
||||
Is the value of the `DATA_TAG` key from the configuration above.
|
||||
items_cache:
|
||||
Is a list of dictionaries, one for each item in the XML file.
|
||||
"""
|
||||
|
||||
# See note above about leading _ and : characters in the XML value
|
||||
lookup = f"wp_post_meta__{data_tag}"
|
||||
|
||||
for attachment in items_cache:
|
||||
# The id of the cached item used in the filter
|
||||
thumbnail_id = attachment.get("wp:post_id")
|
||||
|
||||
# Filter the imported_pages for only pages that include the
|
||||
# matching thumbnail_id in the wp_post_meta field
|
||||
pages = imported_pages.filter(**{lookup: thumbnail_id})
|
||||
|
||||
if pages.exists():
|
||||
# guid is the url of the image to fetch, the get_or_save_image()
|
||||
# function will fetch the image if it doesn't exist
|
||||
image_url = attachment.get("guid")
|
||||
# fix cases where the /wp prefix is missing from the image url
|
||||
if image_url.startswith("https://studentersamfundet.no/wp-content/uploads/"):
|
||||
image_url = image_url.replace(
|
||||
"https://studentersamfundet.no/wp-content/uploads/",
|
||||
"https://studentersamfundet.no/wp/wp-content/uploads/",
|
||||
)
|
||||
try:
|
||||
image = get_or_save_image(image_url)
|
||||
except Exception as e:
|
||||
print("Error with image", image_url, "associated with pages:", pages)
|
||||
print(e)
|
||||
continue
|
||||
|
||||
print("Attaching header images to pages:", pages)
|
||||
try:
|
||||
pages.update(featured_image=image)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
pages.update(logo=image)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -1,17 +1,84 @@
|
||||
from django.core.exceptions import FieldDoesNotExist
|
||||
from django.db import models
|
||||
from wagtail.models import Page
|
||||
|
||||
# Field names declared by WPImportedPageMixin. Concrete models that mix it in
|
||||
# get a manager that .defer()s these so they're never loaded by default — the
|
||||
# columns stay in the database (no migration), and any code path that
|
||||
# explicitly reads them still works via Django's lazy-load.
|
||||
WP_IMPORT_FIELDS = (
|
||||
"wp_post_id",
|
||||
"wp_post_type",
|
||||
"wp_link",
|
||||
"wp_raw_content",
|
||||
"wp_processed_content",
|
||||
"wp_block_json",
|
||||
"wp_normalized_styles",
|
||||
"wp_post_meta",
|
||||
)
|
||||
|
||||
|
||||
# https://github.com/wagtail/wagtail-wordpress-import/blob/main/wagtail_wordpress_import/models.py
|
||||
# DJ001 (null=True on string fields) is suppressed: the schema mirrors the
|
||||
# upstream mixin and changing nullability would force a migration we avoid.
|
||||
class WPImportedPageMixin(Page):
|
||||
wp_post_id = models.IntegerField(blank=True, null=True)
|
||||
wp_post_type = models.CharField(max_length=255, blank=True, null=True)
|
||||
wp_link = models.TextField(blank=True, null=True)
|
||||
wp_raw_content = models.TextField(blank=True, null=True)
|
||||
wp_processed_content = models.TextField(blank=True, null=True)
|
||||
wp_block_json = models.TextField(blank=True, null=True)
|
||||
wp_normalized_styles = models.TextField(blank=True, null=True)
|
||||
wp_post_type = models.CharField(max_length=255, blank=True, null=True) # noqa: DJ001
|
||||
wp_link = models.TextField(blank=True, null=True) # noqa: DJ001
|
||||
wp_raw_content = models.TextField(blank=True, null=True) # noqa: DJ001
|
||||
wp_processed_content = models.TextField(blank=True, null=True) # noqa: DJ001
|
||||
wp_block_json = models.TextField(blank=True, null=True) # noqa: DJ001
|
||||
wp_normalized_styles = models.TextField(blank=True, null=True) # noqa: DJ001
|
||||
wp_post_meta = models.JSONField(blank=True, null=True)
|
||||
|
||||
class Meta:
|
||||
abstract = True
|
||||
|
||||
|
||||
class DeferWPFieldsManagerMixin:
|
||||
"""
|
||||
Manager mixin that always .defer()s the wp_* import columns, so they are
|
||||
never SELECTed by default queries. The columns remain in the database;
|
||||
accessing one on an instance still works (Django lazy-loads it).
|
||||
"""
|
||||
|
||||
def get_queryset(self):
|
||||
return super().get_queryset().defer(*WP_IMPORT_FIELDS)
|
||||
|
||||
|
||||
def _resolve_related_model(model, lookup_path):
|
||||
"""Walk a ``foo__bar`` lookup path and return the final related model, or None."""
|
||||
current = model
|
||||
for part in lookup_path.split("__"):
|
||||
try:
|
||||
field = current._meta.get_field(part)
|
||||
except FieldDoesNotExist:
|
||||
return None
|
||||
current = getattr(field, "related_model", None)
|
||||
if current is None:
|
||||
return None
|
||||
return current
|
||||
|
||||
|
||||
class WPAwareQuerySet(models.QuerySet):
|
||||
"""
|
||||
QuerySet whose ``select_related()`` auto-defers wp_* columns when a join
|
||||
targets a WPImportedPageMixin model. Without this, ``select_related``
|
||||
builds a JOIN that ignores the related model's manager and SELECTs every
|
||||
column including the wp_* blobs. Apply via ``objects = WPAwareManager()``
|
||||
on any model that has a ForeignKey into a WPImported page.
|
||||
"""
|
||||
|
||||
def select_related(self, *fields):
|
||||
qs = super().select_related(*fields)
|
||||
defers = [
|
||||
f"{path}__{name}"
|
||||
for path in fields
|
||||
if (related := _resolve_related_model(qs.model, path)) is not None
|
||||
and issubclass(related, WPImportedPageMixin)
|
||||
for name in WP_IMPORT_FIELDS
|
||||
]
|
||||
return qs.defer(*defers) if defers else qs
|
||||
|
||||
|
||||
WPAwareManager = models.Manager.from_queryset(WPAwareQuerySet)
|
||||
|
||||
Reference in New Issue
Block a user