"
+ for locale in settings.MDN_LANGUAGES:
+ queryset = (Document.objects
+ .filter(is_template=False, locale=locale)
+ .exclude(title__startswith='User:')
+ .exclude(title__iregex=r'Redirect [0-9]+$')
+ .exclude(html__iregex=r'^()?(#)?REDIRECT')
+ .exclude(slug__icontains='Talk:')
+ )
+ if len(queryset) > 0:
+ info = {'queryset': queryset, 'date_field': 'modified'}
+ sitemap = GenericSitemap(info, priority=0.5)
+ urls = sitemap.get_urls(page=1)
+ xml = smart_str(loader.render_to_string('sitemap.xml',
+ {'urlset': urls}))
+ xml = xml.replace('http://', 'https://')
+ directory = '%s/sitemaps/%s' % (settings.MEDIA_ROOT, locale)
+ if not os.path.exists(directory):
+ os.makedirs(directory)
+ f = open('%s/sitemap.xml' % directory, 'w')
+ f.write(xml)
+ f.close()
+
+ sitemap_url = ("https://%s/sitemaps/%s/sitemap.xml" % (
+ Site.objects.get_current().domain, locale))
+ sitemap_index = sitemap_index + sitemap_element % (sitemap_url,
+ time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime()))
+
+ sitemap_index = sitemap_index + "
"
+ index_file = open('%s/sitemap.xml' % settings.MEDIA_ROOT, 'w')
+ index_file.write(parseString(sitemap_index).toxml())
+ index_file.close()
diff --git a/apps/wiki/sitemap.py b/apps/wiki/sitemap.py
deleted file mode 100644
index a53a10c6c60..00000000000
--- a/apps/wiki/sitemap.py
+++ /dev/null
@@ -1,16 +0,0 @@
-from django.contrib.sitemaps import Sitemap
-from wiki.models import (Document, Revision)
-
-class DocumentSitemap(Sitemap):
- changefreq = 'weekly'
- priority = 0.5
-
- def items(self):
- docs = Document.objects.filter(is_template=False)
- return docs
-
- def lastmod(self, doc):
- return doc.current_revision.created
-
- def location(self, doc):
- return doc.get_absolute_url()
\ No newline at end of file
diff --git a/configs/htaccess-without-mindtouch b/configs/htaccess-without-mindtouch
index 6546c2df4f5..e86178b04ba 100644
--- a/configs/htaccess-without-mindtouch
+++ b/configs/htaccess-without-mindtouch
@@ -14,6 +14,11 @@ RewriteRule ^devnews(.*) data/www/devnews$1 [L]
RewriteRule ^web-tech(.*) data/www/web-tech$1 [L]
RewriteRule ^css(.*) data/www/css$1 [L]
+# Rewrites to robots & sitemaps
+RewriteRule ^robots.txt$ media/robots.txt [L]
+RewriteRule ^sitemap.xml$ media/sitemap.xml [L]
+RewriteRule ^sitemaps/([\w\-]*)/sitemap.xml$ media/sitemaps/$1/sitemap.xml [L]
+
# Some blanket section moves / renames
RewriteRule ^En/JavaScript/Reference/Objects/Array$ en-US/docs/JavaScript/Reference/Global_Objects/Array [R=301,L,NC]
RewriteRule ^En/JavaScript/Reference/Objects$ en-US/docs/JavaScript/Reference/Global_Objects/Object [R=301,L,NC]
diff --git a/media/robots.txt b/media/robots.txt
new file mode 100644
index 00000000000..ab7c680d482
--- /dev/null
+++ b/media/robots.txt
@@ -0,0 +1,10 @@
+User-Agent: *
+Crawl-delay: 5
+Sitemap: sitemap.xml
+Request-rate: 1/5
+
+Disallow: /*feed=rss
+Disallow: /*type=feed
+Disallow: /skins
+Disallow: /template:
+Disallow: /media
diff --git a/puppet/files/etc/httpd/conf.d/mozilla-kuma-apache.conf b/puppet/files/etc/httpd/conf.d/mozilla-kuma-apache.conf
index 5cc3af158bd..3f702e80ae3 100644
--- a/puppet/files/etc/httpd/conf.d/mozilla-kuma-apache.conf
+++ b/puppet/files/etc/httpd/conf.d/mozilla-kuma-apache.conf
@@ -21,19 +21,12 @@ WSGISocketPrefix /var/run/wsgi
ServerAlias developer-mdndev.mozilla.org
ServerAlias developer-dev.mozilla.org
- DirectoryIndex index.php index.html
+ DirectoryIndex index.html
Options -Indexes
RewriteEngine On
- DocumentRoot /var/www/dekiwiki
-
-