Fedmsg notify

config/plugins/Makefile.am

@@ -6,6 +6,7 @@ pluginconf_DATA = \
     create-problems.conf \
     fedora.conf \
     fedorabz.conf \
+    fedmsg.conf \
     java.conf \
     kerneloops.conf \
     pull-reports.conf \

config/plugins/fedmsg.conf

@@ -0,0 +1,5 @@
+[fedmsg]
+# you must have the endpoint configured
+name = fedora-infrastructure
+# enviroment can be one of "prod", "stg" or "dev"
+environment = dev

config/plugins/web2.conf

@@ -3,6 +3,8 @@ debug = false
 proxy_setup = false
 secret_key = @SECRET_KEY@
 url = https://example.org/faf/
+# server name is the bare URL without protocols and trailing slash
+server_name = example.org/faf
 brand_title = FAF
 brand_subtitle = Fedora Analysis Framework
 

faf.spec.in

@@ -409,6 +409,14 @@ Requires: %{name}-bugtracker-centos-mantis = %{faf_version}
 %description action-attach-centos-bugs
 A plugin for %{name} implementing attaching of bugs from CentOS Mantis bugtracker
 
+%package action-fedmsg-notify
+Summary: %{name}'s fedmsf-notify plugin
+Requires: %{name} = %{faf_version}
+Requires: fedmsg
+
+%description action-fedmsg-notify
+A plugin for %{name} implementing notification into Fedmsg
+
 %package bugtracker-bugzilla
 Summary: %{name}'s bugzilla support
 Requires: %{name} = %{faf_version}
@@ -1064,6 +1072,10 @@ fi
 %files action-attach-centos-bugs
 %{python_sitelib}/pyfaf/actions/attach_centos_bugs.py*
 
+%files action-fedmsg-notify
+%{python_sitelib}/pyfaf/actions/fedmsg_notify.py*
+%config(noreplace) %{_sysconfdir}/faf/plugins/fedmsg.conf
+
 %files bugtracker-bugzilla
 %{python_sitelib}/pyfaf/bugtrackers/bugzilla.py*
 

src/pyfaf/actions/Makefile.am

@@ -18,6 +18,7 @@ actions_PYTHON = \
     extfaflink.py \
     extfafmodify.py \
     extfafshow.py \
+    fedmsg_notify.py \
     find_components.py \
     find_crash_function.py \
     init.py \

src/pyfaf/actions/fedmsg_notify.py

@@ -0,0 +1,174 @@
+# Copyright (C) 2015  ABRT Team
+# Copyright (C) 2015  Red Hat, Inc.
+#
+# This file is part of faf.
+#
+# faf is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# faf is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with faf.  If not, see <http://www.gnu.org/licenses/>.
+
+from pyfaf.actions import Action
+import datetime
+import argparse
+
+from pyfaf.storage import Report, ReportHistoryDaily, Problem
+from sqlalchemy import func, or_, and_
+import fedmsg
+from pyfaf.utils import web
+
+
+class FedmsgNotify(Action):
+    name = "fedmsg-notify"
+
+    def __init__(self):
+        super(FedmsgNotify, self).__init__()
+        self.load_config_to_self("fedmsg_name", ["fedmsg.name"],
+                                 "fedora-infrastructure")
+        self.load_config_to_self("fedmsg_environment", ["fedmsg.environment"],
+                                 "dev")
+
+    def run(self, cmdline, db):
+        levels = tuple(10**n for n in range(7))
+        fedmsg.init(name=self.fedmsg_name, environment=self.fedmsg_environment)
+        if cmdline.reports:
+            # Sum of counts until yesterday
+            q_yesterday = (
+                db.session
+                .query(Report.id.label("y_report_id"),
+                       func.sum(ReportHistoryDaily.count).label("sum_yesterday"))
+                .outerjoin(ReportHistoryDaily)
+                .filter(ReportHistoryDaily.day < cmdline.date)
+                .group_by(Report.id)
+                .subquery()
+                )
+            # Sum of counts until today
+            q_today = (
+                db.session
+                .query(Report.id.label("t_report_id"),
+                       func.sum(ReportHistoryDaily.count).label("sum_today"))
+                .outerjoin(ReportHistoryDaily)
+                .filter(ReportHistoryDaily.day <= cmdline.date)
+                .group_by(Report.id)
+                .subquery()
+                )
+            q = (db.session.query(Report,
+                                  q_today.c.sum_today,
+                                  q_yesterday.c.sum_yesterday)
+                 .outerjoin(q_today, Report.id == q_today.c.t_report_id)
+                 .outerjoin(q_yesterday, Report.id == q_yesterday.c.y_report_id)
+                 .filter(or_(and_(q_yesterday.c.sum_yesterday == None,
+                                  q_today.c.sum_today != None),
+                             q_today.c.sum_today != q_yesterday.c.sum_yesterday))
+                 )
+
+            for db_report, sum_today, sum_yesterday in q.yield_per(100):
+                # avoid None
+                sum_yesterday = sum_yesterday or 0
+
+                for level in levels:
+                    if sum_yesterday < level and sum_today >= level:
+                        self.log_info("Notifying about report #{0} level {1}"
+                                      .format(db_report.id, level))
+                        msg = {
+                            "report_id": db_report.id,
+                            "function": db_report.crash_function,
+                            "components": [db_report.component.name],
+                            "first_occurrence": db_report.first_occurrence
+                                                .strftime("%Y-%m-%d"),
+                            "count": sum_today,
+                            "type": db_report.type,
+                            "level": level,
+                        }
+                        if web.webfaf_installed():
+                            msg["url"] = web.reverse("reports.item",
+                                                     report_id=db_report.id)
+                        if db_report.problem_id:
+                            msg["problem_id"] = db_report.problem_id
+
+                        fedmsg.publish(
+                            topic="report.threshold{0}".format(level),
+                            modname='faf',
+                            msg=msg)
+
+        if cmdline.problems:
+            # Sum of counts until yesterday
+            q_yesterday = (
+                db.session
+                .query(Problem.id.label("y_problem_id"),
+                       func.sum(ReportHistoryDaily.count).label("sum_yesterday"))
+                .join(Report)
+                .outerjoin(ReportHistoryDaily)
+                .filter(ReportHistoryDaily.day < cmdline.date)
+                .group_by(Problem.id)
+                .subquery()
+                )
+            # Sum of counts until today
+            q_today = (
+                db.session
+                .query(Problem.id.label("t_problem_id"),
+                       func.sum(ReportHistoryDaily.count).label("sum_today"))
+                .join(Report)
+                .outerjoin(ReportHistoryDaily)
+                .filter(ReportHistoryDaily.day <= cmdline.date)
+                .group_by(Problem.id)
+                .subquery()
+                )
+            q = (db.session
+                 .query(Problem, q_today.c.sum_today, q_yesterday.c.sum_yesterday)
+                 .outerjoin(q_today, Problem.id == q_today.c.t_problem_id)
+                 .outerjoin(q_yesterday, Problem.id == q_yesterday.c.y_problem_id)
+                 .filter(or_(and_(q_yesterday.c.sum_yesterday == None,
+                                  q_today.c.sum_today != None),
+                             q_today.c.sum_today != q_yesterday.c.sum_yesterday))
+                 )
+
+            for db_problem, sum_today, sum_yesterday in q.yield_per(100):
+                # avoid None
+                sum_yesterday = sum_yesterday or 0
+
+                for level in levels:
+                    if sum_yesterday < level and sum_today >= level:
+                        self.log_info("Notifying about problem #{0} level {1}"
+                                      .format(db_problem.id, level))
+                        msg = {
+                            "problem_id": db_problem.id,
+                            "function": db_problem.crash_function,
+                            "components": db_problem.unique_component_names,
+                            "first_occurrence": db_problem.first_occurrence
+                                                .strftime("%Y-%m-%d"),
+                            "count": sum_today,
+                            "type": db_problem.type,
+                            "level": level,
+                        }
+                        if web.webfaf_installed():
+                            msg["url"] = web.reverse("problems.item",
+                                                     problem_id=db_problem.id)
+                        fedmsg.publish(
+                            topic="problem.threshold{0}".format(level),
+                            modname='faf',
+                            msg=msg)
+
+    def tweak_cmdline_parser(self, parser):
+        def valid_date(s):
+            try:
+                return datetime.datetime.strptime(s, "%Y-%m-%d").date()
+            except ValueError:
+                msg = "Not a valid date: '{0}'.".format(s)
+                raise argparse.ArgumentTypeError(msg)
+        parser.add_argument("--date",
+                            default=datetime.date.today().strftime("%Y-%m-%d"),
+                            type=valid_date,
+                            help="Date")
+        parser.add_argument("--reports", action="store_true",
+                            default=False, help="Notify about reports")
+        parser.add_argument("--problems", action="store_true",
+                            default=False, help="Notify about problems")

src/pyfaf/actions/stats.py

@@ -105,8 +105,8 @@ def components(self, cmdline, db, opsys, release):
 
                 problem_url = ""
                 if webfaf_installed():
-                    problem_url = reverse("webfaf.problems.views.item",
-                                          args=[report.problem.id])
+                    problem_url = reverse("problems.item",
+                                          problem_id=report.problem.id)
 
                 attached_reports.append((problem_url, report.bugs))
 
@@ -298,7 +298,7 @@ def problems(self, cmdline, db, opsys, release):
             out += "\n"
             if webfaf_installed():
                 out += "URL: "
-                out += reverse("webfaf.problems.views.hot")
+                out += reverse("problems.list")
                 out += "\n\n"
 
         lt = query_longterm_problems(db, release_ids, history=self.history_type)
@@ -309,11 +309,7 @@ def problems(self, cmdline, db, opsys, release):
         if lt:
             out += "Long-term problems:\n\n"
             out += self._render_problems(lt, cmdline.count, release_ids)
-            out += "\n"
-            if webfaf_installed():
-                out += "URL: "
-                out += reverse("webfaf.problems.views.longterm")
-                out += "\n\n"
+            out += "\n\n"
 
         return out
 
@@ -384,8 +380,8 @@ def text_overview(self, cmdline, db, opsys, release):
 
             if webfaf_installed():
                 for report in reports[:3]:
-                    out += "{0}\n".format(reverse("webfaf.reports.views.bthash_forward",
-                                          args=[report.hashes[0].hash]))
+                    out += "{0}\n".format(reverse("reports.bthash_forward",
+                                                  bthash=report.hashes[0].hash))
                     for bug in report.bugs:
                         out += "  {0}\n".format(bug.url)
             else:

src/pyfaf/utils/web.py

@@ -1,7 +1,6 @@
 import os
 import logging
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "webfaf.settings")
+from pyfaf.config import config
 
 
 def webfaf_installed():
@@ -10,8 +9,7 @@ def webfaf_installed():
     """
 
     try:
-        from django.contrib.sites.models import Site
-        Site.objects.get_current()
+        import webfaf2
         return True
     except:
         return False
@@ -23,25 +21,37 @@ def server_url():
     """
 
     if webfaf_installed():
-        from django.contrib.sites.models import Site
-        site = Site.objects.get_current()
-        return "http://" + site.domain
+        return config.get("hub2.url", None)
     else:
         logging.warn("Unable to get web server URL, webfaf not available")
         return None
 
 
-def reverse(view, *args, **kwargs):
+def server_name():
+    """
+    Return web server root URL if applicable
+    """
+
+    if webfaf_installed():
+        return config.get("hub2.server_name", None)
+    else:
+        logging.warn("Unable to get web server name, webfaf not available")
+        return None
+
+
+def reverse(view, **kwargs):
     """
     Return full URL to pointing to `view`
     Wrapper around django"s own reverse.
     """
 
     if webfaf_installed():
-        mainurl = server_url()
-        from django.core.urlresolvers import reverse
-
-        return mainurl + reverse(view, *args, **kwargs)
+        from flask import url_for
+        from webfaf2.webfaf2_main import app
+        app.config["SERVER_NAME"] = server_name()
+        with app.app_context():
+            kwargs["_external"] = True
+            return url_for(view, **kwargs)
     else:
         logging.warn("Unable to get web server URL, webfaf not available")
         return None

src/webfaf2/webfaf2_main.py

@@ -16,11 +16,11 @@
 app = Flask(__name__)
 
 if "WEBFAF_ENVIRON_PRODUCTION" in os.environ:
-    app.config.from_object('config.ProductionConfig')
+    app.config.from_object("webfaf2.config.ProductionConfig")
 elif "WEBFAF_ENVIRON_TEST" in os.environ:
-    app.config.from_object('config.TestingConfig')
+    app.config.from_object("webfaf2.config.TestingConfig")
 else:
-    app.config.from_object('config.DevelopmentConfig')
+    app.config.from_object("webfaf2.config.DevelopmentConfig")
 
 db = SQLAlchemy(app)
 

tests/faftests/test_config.conf

@@ -16,6 +16,7 @@ UrlPrefix = ""
 debug = True
 proxy_setup = False
 secret_key = "NO."
+server_name = example.org/faf
 
 [DumpDir]
 CacheDirectory = /tmp/faf_test_data/dumpdirs/