Merge remote-tracking branch 'sse/develop' into develop

This merges the SSE project into the main FMN repository.

.gitignore

@@ -2,10 +2,8 @@ dist
 build
 *.pyc
 *.egg*
-
-# Ignore the repos checked out within this repo
-fmn.web
-fmn.sse
+_trial_temp/*
+sse.log
 
 # Images are generated as part of the docs build process
 docs/images/*

.travis.yml

@@ -13,6 +13,7 @@ before_install:
 install:
  - pip install --upgrade setuptools pip
  - pip install -r dev-requirements.txt
+ - pip install -r requirements.txt
  - pip install -e .
 script: python -m unittest discover
 

CHANGELOG.rst

@@ -2,6 +2,12 @@
 Changelog
 =========
 
+1.3.0
+=====
+
+* Merge the fmn.sse repository into the fmn repository.
+
+
 1.2.1
 =====
 

MANIFEST.in

@@ -7,4 +7,5 @@ include README.rst
 include requirements.txt
 include dev-requirements.txt
 recursive-include alembic *
+include usr/share/fmn.sse/sse_server.tac
 recursive-include systemd *

dev-data.py

@@ -0,0 +1,42 @@
+import json
+import uuid
+from datetime import datetime as dt
+from random import randint
+import pytz
+import six
+from fmn.sse.FeedQueue import FeedQueue
+
+
+def create_fake_message(additional_text=''):
+    dom_id = six.text_type(uuid.uuid4())
+
+    output = {"dom_id": str(dom_id),
+              "date_time": dt.utcnow().replace(
+                  tzinfo=pytz.utc).isoformat(),
+              "icon": "https://seccdn.libravatar.org/avatar/"
+                      "ba1882fd5522d16213b0535934f77b796f0b89f76edd65460078099"
+                      "fe97c20ea?s=64&d=retro",
+              "link": "localhost:8080",
+              "markup": "New message added to the queue.\t" + additional_text,
+              "secondary_icon": "https://secure.gravatar.com/avatar/"
+                                "ba1882fd5522d16213b0535934f77b796f0b89f76edd"
+                                "65460078099fe97c20ea.jpg?s=64&d"
+              }
+
+    return json.dumps(output)
+
+
+def push_data(fq, number_messages=1000):
+    for i in range(number_messages):
+        msg = "time is: " + str(dt.now().time()) + \
+              "  here's a random number: " + str(randint(0, i))
+        formatted_message = create_fake_message(msg)
+        fq.push_message(formatted_message)
+
+
+fq = FeedQueue(queue_name='skrzepto.id.fedoraproject.org', exchange='user')
+push_data(fq)
+fq = FeedQueue(queue_name='bob.id.fedoraproject.org', exchange='user')
+push_data(fq)
+
+print('finished pushing messages')

docs/sse.rst

@@ -0,0 +1,54 @@
+======================
+FMN Server-Sent Events
+======================
+
+The FMN Server-Sent Events server allows users to view their fedmsg feed in realtime
+using [server-sent events](https://html.spec.whatwg.org/multipage/comms.html#server-sent-events).
+It relies on a service to populate the RabbitMQ message queues for it. Typically, this is
+done with the FMN core services.
+
+Running
+=======
+
+1. Ensure RabbitMQ is running:
+```
+sudo systemctl start rabbitmq-server
+```
+
+2. Start the SSE server:
+```
+twistd -n -l - -y usr/share/fmn.sse/server.tac
+```
+
+3. Make sure the server is available. This should return a HTTP 404:
+```
+curl -v http://localhost:8080/
+```
+
+Test Data
+=========
+
+::
+  $ pip install pytz
+  $ python dev-data.py
+
+Common issues
+=============
+
+Q: I can't connect to rabbitmq with pika
+
+A: Make sure you are running rabbitmq `sudo systemctl start rabbitmq-server`
+
+Q: I get the following error
+
+::
+  pika.exceptions.ChannelClosed: (406, "PRECONDITION_FAILED - inequivalent arg 'x-message-ttl' for queue 'skrzepto.id.fedoraproject.org' in vhost '/': received '60000' but current is '86400000'")
+
+A: You have set the queue a ttl that is not the same. You need to either match the ttl or delete the queue and retry.
+
+Go into http://localhost:15672/  and delete the queue. Thats assuming you enabled the management plugin https://www.rabbitmq.com/management.html
+
+Q: Nothing is being displayed on the curl
+
+A: Wait a few more seconds, it takes a moment to display the data. If it's more
+than a minute check to see if the queue has data via the web ui http://localhost:15672/

fedmsg.d/logging.py

@@ -23,11 +23,22 @@
                 "propagate": False,
                 "handlers": ["console"],
             },
+            fmn={
+                "level": "DEBUG",
+                "propagate": False,
+                "handlers": ["console"],
+            },
             moksha={
                 "level": "DEBUG",
                 "propagate": False,
                 "handlers": ["console"],
             },
         ),
+        # The root logger configuration; this is a catch-all configuration
+        # that applies to all log messages not handled by a different logger
+        root={
+            'level': 'INFO',
+            'handlers': ['console'],
+        },
     ),
 )

fedmsg.d/sse.py

@@ -0,0 +1,35 @@
+config = {
+
+    # SSE
+    "fmn.sse.pika.host": "localhost",
+    "fmn.sse.pika.port": 5672,
+    "fmn.sse.pika.msg_expiration": 3600000,  # 1 hour in ms
+
+    # SSE Web server configuration
+    "fmn.sse.webserver.tcp_port": 8080,
+    # A list of interfaces to listen to ('127.0.0.1', for example); if none
+    # are specified the server listens on all available interfaces.
+    'fmn.sse.webserver.interfaces': [],
+
+    # A regular expression using the standard Python re syntax that defines a
+    # whitelist of queues exposed by the SSE server.
+    'fmn.sse.webserver.queue_whitelist': '.+\.id\.fedoraproject\.org$',
+
+    # A regular expression using the standard Python re syntax that defines a
+    # blacklist for queues exposed by the SSE server. Any queue name that is
+    # matched by the regular expression will return a HTTP 403 to the client.
+    #
+    # Note: This is applied _after_ the whitelist so if the queue is matched
+    # by both regular expressions, the queue _will not_ be served.
+    'fmn.sse.webserver.queue_blacklist': None,
+
+    # The value to use with the 'Access-Control-Allow-Origin' HTTP header
+    'fmn.sse.webserver.allow_origin': '*',
+
+    # Define how many messages to prefetch from the AMQP server
+    'fmn.sse.pika.prefetch_count': 5,
+
+    'endpoints': {
+        # Just need this entry here for tests to pass on travis-ci.
+    }
+}

fmn/sse/__init__.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2017 Jeremy Cline
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
+"""
+Provides a simple Server-Sent events server written in Twisted.
+"""