Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

WIP: Initial message schema for Anitya #570

Merged
merged 3 commits into from
Jan 30, 2019
Merged

WIP: Initial message schema for Anitya #570

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a8cc2f4
Initial message schema for Anitya
jeremycline Aug 1, 2018
a893d6e
Publish messages via fedora_messaging
jeremycline Oct 15, 2018
02c7208
Merge branch 'master' into message-schema
Jan 28, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .coveragerc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
branch = True
include =
anitya/*
anitya_schema/anitya_schema/*


[report]
fail_under = 88
Expand All @@ -13,3 +15,4 @@ omit =
anitya/tests/*
anitya/db/migrations
anitya/db/migrations/*
anitya_schema/anitya_schema/tests/*
10 changes: 5 additions & 5 deletions anitya/admin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def edit_distro(distro_name):
if name != distro.name:
utilities.log(
Session,
distro=distro,
distro=distro.__json__(),
topic='distro.edit',
message=dict(
agent=flask.g.user.username,
Expand Down Expand Up @@ -89,7 +89,7 @@ def delete_distro(distro_name):
if form.validate_on_submit():
utilities.log(
Session,
distro=distro,
distro=distro.__json__(),
topic='distro.remove',
message=dict(
agent=flask.g.user.username,
Expand Down Expand Up @@ -129,7 +129,7 @@ def delete_project(project_id):
if confirm:
utilities.log(
Session,
project=project,
project=project.__json__(),
topic='project.remove',
message=dict(
agent=flask.g.user.username,
Expand Down Expand Up @@ -184,7 +184,7 @@ def delete_project_mapping(project_id, distro_name, pkg_name):
if confirm:
utilities.log(
Session,
project=project,
project=project.__json__(),
topic='project.map.remove',
message=dict(
agent=flask.g.user.username,
Expand Down Expand Up @@ -239,7 +239,7 @@ def delete_project_version(project_id, version):
if confirm:
utilities.log(
Session,
project=project,
project=project.__json__(),
topic='project.version.remove',
message=dict(
agent=flask.g.user.username,
Expand Down
1 change: 1 addition & 0 deletions anitya/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@
r'(?:minsrc|src|source|asc|release))?\.(?:tar|t[bglx]z|tbz2|zip)',
# Token for GitHub API
GITHUB_ACCESS_TOKEN=None,
LEGACY_MESSAGING=False, # If True, publish with fedmsg instead of fedora_messaging
)

# Start with a basic logging configuration, which will be replaced by any user-
Expand Down
59 changes: 32 additions & 27 deletions anitya/lib/utilities.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import logging

import sqlalchemy as sa
from fedora_messaging import api, message, exceptions as fm_exceptions
from sqlalchemy import create_engine
from sqlalchemy.exc import SQLAlchemyError, IntegrityError
from sqlalchemy.orm import sessionmaker
Expand All @@ -30,6 +31,7 @@
import arrow

from . import plugins, exceptions
from anitya import config
from anitya.db import models, Base


Expand All @@ -38,19 +40,29 @@

def fedmsg_publish(*args, **kwargs): # pragma: no cover
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it will be good to rename this function to publish

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a interim solution. The most reasonable refactor is likely to get rid of this function entirely. The reason is that different messages almost certainly have different failure scenarios and once you get rid of the fedmsg portion of this function, all it's doing is squashing exceptions.

''' Try to publish a message on the fedmsg bus. '''
# We catch Exception if we want :-p
# pylint: disable=W0703
# Ignore message about fedmsg import
# pylint: disable=F0401
kwargs['modname'] = 'anitya'
kwargs['cert_prefix'] = 'anitya'
kwargs['name'] = 'relay_inbound'
kwargs['active'] = True
try:
import fedmsg
fedmsg.publish(*args, **kwargs)
except Exception as err:
_log.error(str(err))
if config.config["LEGACY_MESSAGING"]:
# We catch Exception if we want :-p
# pylint: disable=W0703
# Ignore message about fedmsg import
# pylint: disable=F0401
kwargs['modname'] = 'anitya'
kwargs['cert_prefix'] = 'anitya'
kwargs['name'] = 'relay_inbound'
kwargs['active'] = True
try:
import fedmsg
fedmsg.publish(*args, **kwargs)
except Exception as err:
_log.error(str(err))
else:
try:
message_class = message.get_class("anitya." + kwargs["topic"])
api.publish(message_class(body=kwargs["msg"]))
except (fm_exceptions.ConnectionException, fm_exceptions.PublishException) as err:
# For now, continue just logging the error. Once the messaging has
# been untangled into SQLAlchemy events, it should probably result
# in an exception and the client should try again later.
_log.error(str(err))


def check_project_release(project, session, test=False):
Expand Down Expand Up @@ -130,7 +142,7 @@ def check_project_release(project, session, test=False):
if publish:
log(
session,
project=project,
project=project.__json__(),
topic="project.version.update",
message=dict(
project=project.__json__(),
Expand Down Expand Up @@ -204,13 +216,6 @@ def log(session, project=None, distro=None, topic=None, message=None):
message=message,
))

models.Log.insert(
session,
user=message['agent'],
project=project,
distro=distro,
description=final_msg)

return final_msg


Expand Down Expand Up @@ -293,7 +298,7 @@ def create_project(

log(
session,
project=project,
project=project.__json__(),
topic='project.add',
message=dict(
agent=user_id,
Expand Down Expand Up @@ -355,7 +360,7 @@ def edit_project(
if changes:
log(
session,
project=project,
project=project.__json__(),
topic='project.edit',
message=dict(
agent=user_id,
Expand Down Expand Up @@ -412,7 +417,7 @@ def map_project(

log(
session,
distro=distro_obj,
distro=distro_obj.__json__(),
topic='distro.add',
message=dict(
agent=user_id,
Expand Down Expand Up @@ -491,8 +496,8 @@ def map_project(

log(
session,
project=project,
distro=distro_obj,
project=project.__json__(),
distro=distro_obj.__json__(),
topic=topic,
message=message,
)
Expand Down Expand Up @@ -522,7 +527,7 @@ def flag_project(session, project, reason, user_email, user_id):

log(
session,
project=project,
project=project.__json__(),
topic='project.flag',
message=dict(
agent=user_id,
Expand Down
59 changes: 27 additions & 32 deletions anitya/tests/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,13 @@
from __future__ import print_function

from contextlib import contextmanager
import collections
import unittest
import os
import mock

from flask import request_started
from fedora_messaging import message
from social_flask_sqlalchemy.models import PSABase
from sqlalchemy import create_engine, event
import flask_login
Expand Down Expand Up @@ -177,68 +180,66 @@ def create_distro(session):

def create_project(session):
""" Create some basic projects to work with. """
utilities.create_project(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm glad to see you are getting rid of the utilities functions. I want to remove most of them in the future.

session,
project = models.Project(
name='geany',
homepage='https://www.geany.org/',
backend='custom',
version_url='https://www.geany.org/Download/Releases',
regex='DEFAULT',
user_id='noreply@fedoraproject.org',
)
session.add(project)

utilities.create_project(
session,
project = models.Project(
name='subsurface',
homepage='https://subsurface-divelog.org/',
backend='custom',
version_url='https://subsurface-divelog.org/downloads/',
regex='DEFAULT',
user_id='noreply@fedoraproject.org',
)
session.add(project)

utilities.create_project(
session,
project = models.Project(
name='R2spec',
homepage='https://fedorahosted.org/r2spec/',
user_id='noreply@fedoraproject.org',
backend='custom',
)
session.add(project)
session.commit()


def create_ecosystem_projects(session):
""" Create some fake projects from particular upstream ecosystems

Each project name is used in two different ecosystems
"""
utilities.create_project(
session,
project = models.Project(
name='pypi_and_npm',
homepage='https://example.com/not-a-real-pypi-project',
backend='PyPI',
user_id='noreply@fedoraproject.org'
)
session.add(project)

utilities.create_project(
session,
project = models.Project(
name='pypi_and_npm',
homepage='https://example.com/not-a-real-npmjs-project',
backend='npmjs',
user_id='noreply@fedoraproject.org'
)
session.add(project)

utilities.create_project(
session,
project = models.Project(
name='rubygems_and_maven',
homepage='https://example.com/not-a-real-rubygems-project',
backend='Rubygems',
user_id='noreply@fedoraproject.org'
)
session.add(project)

utilities.create_project(
session,
project = models.Project(
name='rubygems_and_maven',
homepage='https://example.com/not-a-real-maven-project',
backend='Maven Central',
user_id='noreply@fedoraproject.org'
)
session.add(project)
session.commit()


def create_package(session):
Expand All @@ -262,25 +263,19 @@ def create_package(session):

def create_flagged_project(session):
""" Create and flag a project. Returns the ProjectFlag. """
project = utilities.create_project(
session,
project = models.Project(
name='geany',
homepage='https://www.geany.org/',
version_url='https://www.geany.org/Download/Releases',
regex='DEFAULT',
user_id='noreply@fedoraproject.org',
)

session.add(project)

flag = utilities.flag_project(
session,
project,
"This is a duplicate.",
"dgay@redhat.com",
"user_openid_id",
flag = models.ProjectFlag(
project=project,
reason="this is a duplicate.",
user="dgay@redhat.com",
)

session.add(flag)

session.commit()
Expand Down
2 changes: 2 additions & 0 deletions anitya/tests/test_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@

default_regex = "a*b*"
github_access_token = "foobar"
legacy_messaging = true

[anitya_log_config]
version = 1
Expand Down Expand Up @@ -171,6 +172,7 @@ def test_full_config_file(self, mock_exists, mock_log):
'LIBRARIESIO_PLATFORM_WHITELIST': ['pypi', 'rubygems'],
'DEFAULT_REGEX': 'a*b*',
'GITHUB_ACCESS_TOKEN': 'foobar',
'LEGACY_MESSAGING': True,
}
config = anitya_config.load()
self.assertEqual(sorted(expected_config.keys()), sorted(config.keys()))
Expand Down
Loading