Bug 896474: Generate forums from data file in SVN.

bedrock/externalfiles/__init__.py

@@ -68,7 +68,7 @@ def last_modified_callback(self, *args, **kwargs):
     def validate_resp(self, resp):
         """
         Validate the response from the server, returning the content.
-        :param contents: requests.Response
+        :param resp: requests.Response
         :return: str or None
         :raises: ValueError
         """

bedrock/mozorg/forums.py

@@ -0,0 +1,59 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import re
+
+from django.utils.functional import cached_property
+
+from ordereddict import OrderedDict
+
+from bedrock.externalfiles import ExternalFile
+
+
+class ForumsFile(ExternalFile):
+    forum_line_re = re.compile(r'^((?:mozilla\.)?[a-z0-9\-\.]+)\s+(.*)$')
+    title_line_re = re.compile(r'^:(.*)$')
+
+    def validate_content(self, content):
+        lines = content.split('\n')
+        try:
+            forums = self._parse(lines)
+        except Exception:
+            raise ValueError('Error parsing forums file.')
+
+        # currently 15 categories
+        if not len(forums.keys()) > 10:
+            raise ValueError('Forums file truncated or corrupted.')
+
+        return content
+
+    @cached_property
+    def ordered(self):
+        return self._parse(self.readlines())
+
+    def _parse(self, lines):
+        forums = OrderedDict()
+        current_group = None
+        for line in lines:
+            line = line.strip()
+            if not line:
+                continue
+
+            title_m = self.title_line_re.match(line)
+            if title_m:
+                current_group = forums[title_m.group(1)] = []
+
+            elif current_group is not None:
+                forum_m = self.forum_line_re.match(line)
+                if forum_m:
+                    forum_id, forum_desc = forum_m.groups()
+                    forum_id_dashed = re.sub(r'^mozilla\.', '', forum_id)
+                    forum_id_dashed = forum_id_dashed.replace('.', '-')
+                    current_group.append({
+                        'id': forum_id,
+                        'dashed': forum_id_dashed,
+                        'description': forum_desc,
+                    })
+
+        return forums

bedrock/mozorg/helpers/misc.py

@@ -3,6 +3,7 @@
 from os import path
 
 from django.conf import settings
+from django.template.defaultfilters import slugify as django_slugify
 
 import jingo
 import jinja2
@@ -497,3 +498,13 @@ def shuffle(_list):
     """Return a shuffled list"""
     random.shuffle(_list)
     return _list
+
+
+@jingo.register.filter
+def slugify(text):
+    """
+    Converts to lowercase, removes non-word characters (alphanumerics and
+    underscores) and converts spaces to hyphens. Also strips leading and
+    trailing whitespace.
+    """
+    return django_slugify(text)

bedrock/mozorg/templates/mozorg/about/forums/forums.html

@@ -48,16 +48,9 @@ <h1 class="title-shadow-box">{{ self.page_title() }}</h1>
   <nav class="toc">
     <h3>{{ _('Categories:') }}</h3>
     <ul>
-      <li><a href="#end-user-support">{{ _('End-User Support') }}</a></li>
-      <li><a href="#projectwide-forums">{{ _('Projectwide Forums') }}</a></li>
-      <li><a href="#general-development">{{ _('General Development') }}</a></li>
-      <li><a href="#mozilla-technologies">{{ _('Mozilla Technologies') }}</a></li>
-      <li><a href="#application-development">{{ _('Application Development') }}</a></li>
-      <li><a href="#extension-development">{{ _('Extension Development') }}</a></li>
-      <li><a href="#ports-to-other-oses">{{ _('Ports To Other OSes') }}</a></li>
-      <li><a href="#mozilla-drumbeat">{{ _('Mozilla Drumbeat') }}</a></li>
-      <li><a href="#regional">{{ _('Regional') }}</a></li>
-      <li><a href="#testing">{{ _('Testing') }}</a></li>
+      {% for title in forums.ordered.keys() %}
+        <li><a href="#{{ title|slugify }}">{{ title }}</a></li>
+      {% endfor %}
       <li><a href="#mozilla-labs">{{ _('Mozilla Labs') }}</a></li>
       <li><a href="#chat">{{ _('Realtime Chat') }}</a></li>
     </ul>
@@ -66,11 +59,46 @@ <h3>{{ _('Categories:') }}</h3>
 </section>
 
 <!--BEGIN_LIST-->
-<!-- Generated by gen-ng-info from raw-ng-list.txt -->
 
+{% for title, forums_list in forums.ordered.iteritems() %}
+  <h2 id="{{ title|slugify }}">{{ title }}</h2>
+
+  <p>{{ _('Please read the <a href="%s">Mozilla forum etiquette</a> document for posting rules and conventions.')|format(url('mozorg.about.forums.etiquette')) }}</p>
+
+  <table class="forumlist table">
+    <colgroup><col class="name" /><col /><col class="subscribe" /></colgroup>
+    {% for forum in forums_list %}
+      <tr id="{{ forum.dashed }}">
+        <td>
+          <table>
+            <tr>
+              <th>
+                {{ forum.id }}<sup><small>(<a href="#{{ forum.dashed }}">#</a>)</small></sup>
+              </th>
+            </tr>
+            <tr>
+              <td>
+                {{ forum.description|safe }}
+              </td>
+            </tr>
+          </table>
+        </td>
+        <td>
+          <ul class="subscribe">
+            {% if forum.id.startswith('mozilla.') %}
+              <li>{{ _('Mailing List:') }} <a href="https://lists.mozilla.org/listinfo/{{ forum.dashed }}">{{ forum.dashed }}&#64;lists.mozilla.org</a></li>
+              <li>{{ _('Newsgroup:') }} <a href="news://news.mozilla.org/{{ forum.id }}">{{ forum.id }}</a></li>
+              <li>{{ _('Web:') }} <a href="http://groups.google.com/group/{{ forum.id }}/topics">Google Groups</a></li>
+            {% else %}
+              <li>{{ _('Mailing List:') }} <a href='https://mail.mozilla.org/listinfo/{{ forum.dashed }}'>{{ forum.dashed }}&#64;mozilla.org</a></li>
+            {% endif %}
+          </ul>
+        </td>
+      </tr>
+    {% endfor %}
+  </table>
+{% endfor %}
 
-
-<!-- end of generated HTML -->
 <!--END_LIST-->
 
 <h2 id="mozilla-labs">{{ _('Mozilla Labs') }}</h2>

bedrock/mozorg/tests/test_forums.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from ordereddict import OrderedDict
+
+from bedrock.mozorg import forums
+from bedrock.mozorg.tests import TestCase
+
+
+FORUMS_GOOD_CONTENT = [
+    ':Particularly Important Forums\n',
+    '\n',
+    'mozilla.announce                        New releases will be announced here.\n',
+    'mozilla.announce.compatibility          Add-on compatibility announcements. (Moderated)\n',
+    '\n',
+    ':Applications and Projects\n',
+    '\n',
+    'firefox-dev                             For development of Firefox.\n',
+]
+
+FORUMS_GOOD_DICT = OrderedDict()
+FORUMS_GOOD_DICT['Particularly Important Forums'] = [
+    {'id': 'mozilla.announce', 'dashed': 'announce',
+     'description': 'New releases will be announced here.'},
+    {'id': 'mozilla.announce.compatibility', 'dashed': 'announce-compatibility',
+     'description': 'Add-on compatibility announcements. (Moderated)'},
+]
+FORUMS_GOOD_DICT['Applications and Projects'] = [
+    {'id': 'firefox-dev', 'dashed': 'firefox-dev', 'description': 'For development of Firefox.'},
+]
+
+
+class TestForums(TestCase):
+    def setUp(self):
+        self.forums_file = forums.ForumsFile('forums')
+
+    def test_forums_ordered(self):
+        """Should give an ordered dict of forums from file."""
+        for title, forums_list in self.forums_file._parse(FORUMS_GOOD_CONTENT).items():
+            for i, forum in enumerate(forums_list):
+                self.assertDictEqual(forum, FORUMS_GOOD_DICT[title][i])
+
+    def test_forums_validation(self):
+        with self.assertRaises(ValueError):
+            # shouldn't be enough data
+            self.forums_file.validate_content(''.join(FORUMS_GOOD_CONTENT))

bedrock/mozorg/urls.py

@@ -30,7 +30,7 @@
     page('mission', 'mozorg/mission.html'),
     page('ITU', 'mozorg/itu.html'),
     page('about/powered-by', 'mozorg/powered-by.html'),
-    page('about/forums', 'mozorg/about/forums/forums.html'),
+    url('^about/forums/$', views.forums_view, name='mozorg.about.forums.forums'),
     page('about/forums/etiquette', 'mozorg/about/forums/etiquette.html'),
     page('about/forums/cancellation', 'mozorg/about/forums/cancellation.html'),
     page('about/governance', 'mozorg/about/governance/governance.html'),

bedrock/mozorg/views.py

@@ -29,13 +29,15 @@
 from bedrock.mozorg.forms import (ContributeForm,
                                   ContributeStudentAmbassadorForm,
                                   WebToLeadForm)
+from bedrock.mozorg.forums import ForumsFile
 from bedrock.mozorg.models import TwitterCache
 from bedrock.mozorg.util import hide_contrib_form
 from bedrock.mozorg.util import HttpResponseJSON
 from bedrock.newsletter.forms import NewsletterFooterForm
 
 
 credits_file = CreditsFile('credits')
+forums_file = ForumsFile('forums')
 
 
 def csrf_failure(request, reason=''):
@@ -276,6 +278,15 @@ def credits_view(request):
     return django_render(request, 'mozorg/credits.html', ctx)
 
 
+@cache_control_expires(2)
+@last_modified(forums_file.last_modified_callback)
+@require_safe
+def forums_view(request):
+    """Display our mailing lists and newsgroups."""
+    ctx = {'forums': forums_file}
+    return l10n_utils.render(request, 'mozorg/about/forums/forums.html', ctx)
+
+
 class Robots(TemplateView):
     template_name = 'mozorg/robots.txt'
 

bedrock/settings/base.py

@@ -917,6 +917,11 @@ def lazy_email_backend():
         'type': 'bedrock.mozorg.credits.CreditsFile',
         'name': 'credits.csv',
     },
+    'forums': {
+        'url': 'https://svn.mozilla.org/projects/mozilla.org/trunk/about/forums/raw-ng-list.txt',
+        'type': 'bedrock.mozorg.forums.ForumsFile',
+        'name': 'forums.txt',
+    },
 }
 
 FACEBOOK_LOCALES = ['en-US', 'es-ES', 'pt-BR', 'id', 'de']

docs/install.rst

@@ -60,6 +60,10 @@ Check out the latest product-details::
 
 This pulls in version information for Mozilla products like Firefox.
 
+Next get the data files for certain pages (so far /credits/ and /about/forums/)::
+
+    $ ./manage.py update_externalfiles
+
 Lastly, you need to install `node` and the `less` package. Soon you won't need this for local development but currently it compiles the LESS CSS code on the server-side::
 
     $ npm -g install less

etc/httpd/global.conf

@@ -378,9 +378,6 @@ RewriteRule ^/(\w{2,3}(?:-\w{2})?/)?about/timeline.html$ https://wiki.mozilla.or
 # bug 1016400
 RewriteRule ^/(\w{2,3}(?:-\w{2})?/)?about/careers.html$ https://careers.mozilla.org/ [QSA,L,R=301]
 
-# bug 994831
-RewriteRule ^/(\w{2,3}(?:-\w{2})?/)?about/history/details/$ /b/$1about/history/details/ [PT]
-
 # bug 861243 and bug 869489
 RewriteRule ^/about/manifesto\.html$ /about/manifesto/ [L,R=301]
 RewriteRule ^/about/manifesto\.(.*)\.html$ /$1/about/manifesto/ [L,R=301]
@@ -443,8 +440,6 @@ RewriteRule ^/en-US/about/buttons/(.*) /media/img/careers/buttons/$1 [L,R=301]
 # bug 960689, 1013349, 896474
 RewriteRule ^/(\w{2,3}(?:-\w{2})?/)?about/legal\.html$ /$1about/legal/ [L,R=301]
 RewriteRule ^/(\w{2,3}(?:-\w{2})?/)?about/partnerships\.html$ /$1about/partnerships/ [L,R=301]
-RewriteCond %{REQUEST_URI} !/about/forums/?
-RewriteRule ^/(\w{2,3}(?:-\w{2})?/)?about(.*)$ /b/$1about$2 [PT]
 
 # bug 809426
 RewriteRule ^/(\w{2,3}(?:-\w{2})?/)?legal/eula(/?)$ /b/$1legal/eula$2 [PT]
@@ -773,3 +768,6 @@ RewriteRule ^/(\w{2,3}(?:-\w{2})?/)?firefox/fx(/?)$ /$1firefox/new/ [L,R=301]
 # Bug 1005674
 RewriteCond %{QUERY_STRING} !^flv=(?:https?://videos\.(?:mozilla\.org|cdn\.mozilla\.net))?/\w+/
 RewriteRule ^/media/flash/playerWithControls.swf - [F]
+
+# Bug 896474
+RewriteRule ^/(\w{2,3}(?:-\w{2})?/)?about(/?|/.+)$ /b/$1about$2 [PT]

media/css/mozorg/about-forums.less

@@ -46,6 +46,18 @@
     }
 }
 
+.table table {
+    td {
+        border-top: none;
+        padding-bottom: 0;
+    }
+    th {
+        padding-top: 0;
+        border-top: none;
+        border-bottom: 1px solid rgba(0, 0, 0, 0.2);
+    }
+}
+
 
 /* Tablet Layout: 760px */
 @media only screen and (min-width: @breakTablet) and (max-width: @breakDesktop) {