Add robots meta

Co-authored-by: corentinbettiol <cb@kapt.mobi>

changes/116.feature

@@ -0,0 +1 @@
+Add meta robots

djangocms_page_meta/admin.py

@@ -5,7 +5,7 @@
 from django.contrib import admin
 from django.utils.translation import gettext_lazy as _
 
-from .forms import GenericAttributeInlineForm, TitleMetaAdminForm
+from .forms import GenericAttributeInlineForm, PageMetaAdminForm, TitleMetaAdminForm
 from .models import GenericMetaAttribute, PageMeta, TitleMeta
 
 
@@ -26,6 +26,7 @@ class GenericAttributeTitleInline(admin.TabularInline):
 @admin.register(PageMeta)
 class PageMetaAdmin(PageExtensionAdmin):
     raw_id_fields = ("og_author",)
+    form = PageMetaAdminForm
     inlines = (GenericAttributePageInline,)
     fieldsets = (
         (None, {"fields": ("image",)}),
@@ -42,6 +43,7 @@ class PageMetaAdmin(PageExtensionAdmin):
         ),
         (_("Twitter Cards"), {"fields": ("twitter_type", "twitter_author"), "classes": ("collapse",)}),
         (_("Schema.org microdata"), {"fields": ("schemaorg_type",), "classes": ("collapse")}),
+        (_("Robots"), {"fields": ("robots",), "classes": ("collapse")}),
     )
 
     class Media:

djangocms_page_meta/forms.py

@@ -1,10 +1,27 @@
 from django import forms
 from django.core.validators import MaxLengthValidator
 
-from .models import GenericMetaAttribute, TitleMeta
+from .models import GenericMetaAttribute, PageMeta, TitleMeta
 from .settings import get_setting
 
 
+class PageMetaAdminForm(forms.ModelForm):
+    robots = forms.MultipleChoiceField(
+        choices=get_setting("ROBOTS_CHOICES"),
+        widget=forms.CheckboxSelectMultiple,
+        required=False,
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if kwargs.get("instance"):
+            self.initial["robots"] = kwargs.get("instance").robots_list
+
+    class Meta:
+        model = PageMeta
+        exclude = ()
+
+
 class TitleMetaAdminForm(forms.ModelForm):
     def __init__(self, *args, **kwargs):
         self.base_fields["description"].validators = [MaxLengthValidator(get_setting("DESCRIPTION_LENGTH"))]

djangocms_page_meta/migrations/0014_pagemeta_robots_alter_genericmetaattribute_id_and_more.py

@@ -0,0 +1,32 @@
+# Generated by Django 4.2.4 on 2023-08-03 13:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("djangocms_page_meta", "0013_auto_20200821_1457"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="pagemeta",
+            name="robots",
+            field=models.CharField(blank=True, max_length=512, verbose_name="Robots meta tag"),
+        ),
+        migrations.AlterField(
+            model_name="genericmetaattribute",
+            name="id",
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID"),
+        ),
+        migrations.AlterField(
+            model_name="pagemeta",
+            name="id",
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID"),
+        ),
+        migrations.AlterField(
+            model_name="titlemeta",
+            name="id",
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID"),
+        ),
+    ]

djangocms_page_meta/models.py

@@ -1,3 +1,5 @@
+import ast
+
 from cms.extensions import PageExtension, TitleExtension
 from cms.extensions.extension_pool import extension_pool
 from cms.models import Page, Title
@@ -60,14 +62,14 @@ class PageMeta(PageExtension):
     twitter_type = models.CharField(
         _("Resource type"), max_length=255, choices=meta_settings.TWITTER_TYPES, blank=True
     )
-
     schemaorg_type = models.CharField(
         _("Resource type"),
         max_length=255,
         choices=meta_settings.SCHEMAORG_TYPES,
         blank=True,
         help_text=_("Use Article for generic pages."),
     )
+    robots = models.CharField(_("Robots meta tag"), max_length=512, blank=True)
 
     class Meta:
         verbose_name = _("Page meta info (all languages)")
@@ -86,6 +88,12 @@ def copy_relations(self, oldinstance, language):
             item.page = self
             item.save()
 
+    @property
+    def robots_list(self):
+        if self.robots:
+            return ast.literal_eval(self.robots)
+        return None
+
 
 extension_pool.register(PageMeta)
 

djangocms_page_meta/settings.py

@@ -1,12 +1,27 @@
+from django.utils.translation import gettext_lazy as _
+
+
 def get_setting(name):
     from django.conf import settings
 
     description_length = getattr(settings, "PAGE_META_DESCRIPTION_LENGTH", None) or 320
 
     tw_description_length = getattr(settings, "PAGE_META_TWITTER_DESCRIPTION_LENGTH", None) or 320
 
+    robots_choices = getattr(settings, "PAGE_META_ROBOTS_CHOICES", None) or (
+        ("none", _("None [noindex, nofollow]")),
+        ("noindex", _("No Index")),
+        ("nofollow", _("No Follow")),
+        ("noimageindex", _("No Image Index")),
+        ("nosnippet", _("No Snippet")),
+        ("noarchive", _("No Archive")),
+        ("notranslate", _("No Translate")),
+        ("nositelinkssearchbox", _("No Site Links Search Box")),
+    )
+
     default = {
         "PAGE_META_DESCRIPTION_LENGTH": description_length,
         "PAGE_META_TWITTER_DESCRIPTION_LENGTH": tw_description_length,
+        "PAGE_META_ROBOTS_CHOICES": robots_choices,
     }
     return default["PAGE_META_%s" % name]

djangocms_page_meta/templates/djangocms_page_meta/meta.html

@@ -36,6 +36,7 @@
 		{% if meta.twitter_author %}{% twitter_prop 'creator' meta.twitter_author %}{% endif %}
 		{% if meta.twitter_site %}{% twitter_prop 'site_name' meta.twitter_site %}{% endif %}
 	{% endif %}
+  {% if meta.robots %}{% meta_list 'robots' meta.robots %}{% endif %}
 	{% if meta.use_schemaorg %}
         {% if meta.schemaorg_name %}{% schemaorg_prop 'name' meta.schemaorg_name %}{% endif %}
         {% if meta.schemaorg_datePublished %}{% schemaorg_prop 'datePublished' meta.schemaorg_datePublished %}{% endif %}

djangocms_page_meta/utils.py

@@ -107,6 +107,7 @@ def get_page_meta(page, language):
             meta.twitter_site = pagemeta.twitter_site
             meta.twitter_author = pagemeta.twitter_author
             meta.schemaorg_type = pagemeta.schemaorg_type
+            meta.robots = pagemeta.robots_list
             if page.publication_date:
                 meta.published_time = page.publication_date.isoformat()
             if page.changed_date:

docs/configuration.rst

@@ -16,14 +16,35 @@ PAGE_META_DESCRIPTION_LENGTH
 Set the max length of the HTML meta description field.
 Default is ``320``.
 
-.. PAGE_META_TWITTER_DESCRIPTION_LENGTH:
+.. _PAGE_META_TWITTER_DESCRIPTION_LENGTH:
 
 PAGE_META_TWITTER_DESCRIPTION_LENGTH
 ------------------------------------
 
 Set the max length of the Twitter card description field.
 Default is ``280``.
 
+.. _PAGE_META_ROBOTS_CHOICES:
+
+PAGE_META_ROBOTS_CHOICES
+------------------------
+
+Set all the available choices for robots meta tag.
+Default is:
+
+.. code-block:: python
+
+    (
+        ("none", _("None [noindex, nofollow]")),
+        ("noindex", _("No Index")),
+        ("nofollow", _("No Follow")),
+        ("noimageindex", _("No Image Index")),
+        ("nosnippet", _("No Snippet")),
+        ("noarchive", _("No Archive")),
+        ("notranslate", _("No Translate")),
+        ("nositelinkssearchbox", _("No Site Links Search Box")),
+    )
+
 django-meta configuration
 =========================
 

docs/usage.rst

@@ -60,9 +60,9 @@ Supported attributes
 ********************
 
 ``djangocms-page-meta`` currently offers partial support for `OpenGraph`_,
-`Twitter Cards`_ and `Schema.org microdata`_. As a generic application
-``djangocms-page-meta`` cannot cover every use case while still being
-useful to most people.
+`Twitter Cards`_, `Schema.org microdata`_ and robots meta tag. As a generic
+application ``djangocms-page-meta`` cannot cover every use case while
+still being useful to most people.
 
 
 Generic HTML

tests/__init__.py

@@ -55,6 +55,8 @@ class BaseTest(BaseTestCase):
         "twitter_site": "fake_site",
         "twitter_type": "summary",
     }
+    robots_data_single = {"robots": "['noindex']"}
+    robots_data_multiple = {"robots": "['none', 'noimageindex', 'noarchive']"}
 
     def setUp(self):
         super().setUp()

tests/test_general.py

@@ -73,6 +73,35 @@ def test_page_meta_twitter(self):
         self.assertEqual(meta.twitter_type, self.twitter_data["twitter_type"])
         self.assertEqual(meta.get_domain(), settings.META_SITE_DOMAIN)
 
+    def test_page_meta_robots_no_data(self):
+        page, __ = self.get_pages()
+        page_meta = models.PageMeta.objects.create(extended_object=page)
+        page.reload()
+        meta = get_page_meta(page, "en")
+        self.assertEqual(meta.robots, page_meta.robots_list)
+
+    def test_page_meta_robots_single(self):
+        page, __ = self.get_pages()
+        page_meta = models.PageMeta.objects.create(extended_object=page)
+        for key, val in self.robots_data_single.items():
+            setattr(page_meta, key, val)
+        page_meta.save()
+        page.reload()
+        meta = get_page_meta(page, "en")
+        self.assertEqual(page_meta.robots, self.robots_data_single["robots"])
+        self.assertEqual(meta.robots, page_meta.robots_list)
+
+    def test_page_meta_robots_multiple(self):
+        page, __ = self.get_pages()
+        page_meta = models.PageMeta.objects.create(extended_object=page)
+        for key, val in self.robots_data_multiple.items():
+            setattr(page_meta, key, val)
+        page_meta.save()
+        page.reload()
+        meta = get_page_meta(page, "en")
+        self.assertEqual(page_meta.robots, self.robots_data_multiple["robots"])
+        self.assertEqual(meta.robots, page_meta.robots_list)
+
     def test_none_page(self):
         meta = get_page_meta(None, "en")
         self.assertIsNone(meta)
@@ -179,6 +208,14 @@ def test_str_methods(self):
         self.assertEqual(str(page_attr), f"Attribute {page_attr.name} for {page_meta}")
         self.assertEqual(str(title_attr), f"Attribute {title_attr.name} for {title_meta}")
 
+    def test_robots_list_property(self):
+        page1, __ = self.get_pages()
+        page_meta = models.PageMeta.objects.create(extended_object=page1)
+        self.assertIsNone(page_meta.robots_list)
+        page_meta.robots = "['noindex', 'notranslate', 'nosnippet']"
+        page_meta.save()
+        self.assertEqual(page_meta.robots_list, ["noindex", "notranslate", "nosnippet"])
+
     def test_cache_cleanup_on_update_delete_meta(self):
         """
         Meta caches are emptied when updating / deleting a meta

tests/test_templatetags.py

@@ -33,6 +33,46 @@ def test_page_meta(self):
         self.assertContains(response, '<meta property="article:publisher" content="https://facebook.com/FakeUser">')
         self.assertContains(response, '<meta custom="attr" content="foo">')
 
+    def test_page_meta_robots_no_data(self):
+        """
+        Test page-level no robots templatetags
+        """
+        page1, __ = self.get_pages()
+        page_ext = PageMeta.objects.create(extended_object=page1)
+        page_ext.save()
+        page1.save()
+        page1.publish("en")
+        response = self.client.get(page1.get_public_url("en"))
+        self.assertNotContains(response, '<meta name="robots"')
+
+    def test_page_meta_robots_single(self):
+        """
+        Test page-level robots single templatetags
+        """
+        page1, __ = self.get_pages()
+        page_ext = PageMeta.objects.create(extended_object=page1)
+        for key, val in self.robots_data_single.items():
+            setattr(page_ext, key, val)
+        page_ext.save()
+        page1.save()
+        page1.publish("en")
+        response = self.client.get(page1.get_public_url("en"))
+        self.assertContains(response, '<meta name="robots" content="noindex">')
+
+    def test_page_meta_robots_multiple(self):
+        """
+        Test page-level robots multiple templatetags
+        """
+        page1, __ = self.get_pages()
+        page_ext = PageMeta.objects.create(extended_object=page1)
+        for key, val in self.robots_data_multiple.items():
+            setattr(page_ext, key, val)
+        page_ext.save()
+        page1.save()
+        page1.publish("en")
+        response = self.client.get(page1.get_public_url("en"))
+        self.assertContains(response, '<meta name="robots" content="none, noimageindex, noarchive">')
+
     def test_title_meta(self):
         """
         Test title-level templatetags