Generate sitemaps and fix robots.txt path for analytics

redirects.yaml

@@ -726,7 +726,6 @@ release-notes/precise-pangolin/?: "https://wiki.ubuntu.com/PrecisePangolin/Relea
 releases/?: "https://releases.ubuntu.com"
 releaseendoflife/?: "/about/release-cycle"
 risc-v/?: "https://wiki.ubuntu.com/RISC-V"
-robots.txt?: "/static/files/robots.txt"
 robotics/esm/?: "/robotics/ros-esm"
 rss\.xml/?: "/blog/feed"
 rss\.xmlsubscribe=feed/?: "/blog/feed"

robots.txt

@@ -0,0 +1,4 @@
+User-Agent: *
+Disallow: /search
+Disallow: /search*
+Sitemap: https://ubuntu.com/sitemap.xml

webapp/app.py

@@ -152,6 +152,7 @@
     subscription_centre,
     thank_you,
     unlisted_engage_page,
+    build_engage_pages_sitemap,
 )
 
 DISCOURSE_API_KEY = os.getenv("DISCOURSE_API_KEY")
@@ -537,6 +538,13 @@
     exclude_topics=[17229, 18033, 17250],
 )
 
+
+app.add_url_rule(
+    "/engage/sitemap.xml",
+    view_func=build_engage_pages_sitemap(engage_pages),
+)
+
+
 app.add_url_rule(
     "/openstack/resources", view_func=openstack_engage(engage_pages)
 )

webapp/views.py

@@ -556,6 +556,36 @@ def unlisted_engage_page(slug):
         return flask.abort(404)
 
 
+def build_engage_pages_sitemap(engage_pages):
+    """
+    Create sitemaps for each engage page
+    """
+
+    def ep_sitemap():
+        links = []
+        metadata = engage_pages.get_index()
+        if len(metadata) == 0:
+            flask.abort(404)
+
+        for page in metadata:
+            links.append(
+                {
+                    "url": f'https://ubuntu.com/{page["path"]}',
+                    "last_updated": page["updated"].strftime("%-d %B %Y"),
+                }
+            )
+
+        xml_sitemap = flask.render_template("sitemap.xml", links=links)
+
+        response = flask.make_response(xml_sitemap)
+        response.headers["Content-Type"] = "application/xml"
+        response.headers["Cache-Control"] = "public, max-age=43200"
+
+        return response
+
+    return ep_sitemap
+
+
 def openstack_install():
     """
     OpenStack install docs