Merge pull request #234 from gnaponie/FACTORY-3100

Display failed and in-progress container builds

estuary/models/__init__.py

@@ -4,12 +4,12 @@
 from estuary.models.bugzilla import BugzillaBug
 from estuary.models.distgit import DistGitRepo, DistGitBranch, DistGitCommit
 from estuary.models.errata import Advisory, ContainerAdvisory
-from estuary.models.freshmaker import FreshmakerEvent
+from estuary.models.freshmaker import FreshmakerEvent, FreshmakerBuild
 from estuary.models.user import User
 
 all_models = (Advisory, BugzillaBug, ContainerAdvisory, ContainerKojiBuild,
-              DistGitBranch, DistGitCommit, DistGitRepo, FreshmakerEvent, KojiBuild,
-              KojiTag, ModuleKojiBuild, User)
+              DistGitBranch, DistGitCommit, DistGitRepo, FreshmakerEvent, FreshmakerBuild,
+              KojiBuild, KojiTag, ModuleKojiBuild, User)
 names_to_model = {model.__label__: model for model in all_models}
 
 

estuary/models/freshmaker.py

@@ -3,8 +3,8 @@
 from __future__ import unicode_literals
 
 from neomodel import (
-    UniqueIdProperty, RelationshipTo, IntegerProperty, StringProperty,
-    ZeroOrOne)
+    UniqueIdProperty, RelationshipTo, IntegerProperty, StringProperty, DateTimeProperty,
+    ZeroOrOne, RelationshipFrom)
 
 from estuary.models.base import EstuaryStructuredNode
 
@@ -20,9 +20,36 @@ class FreshmakerEvent(EstuaryStructuredNode):
     state_reason = StringProperty()
     triggered_by_advisory = RelationshipTo(
         '.errata.Advisory', 'TRIGGERED_BY', cardinality=ZeroOrOne)
-    triggered_container_builds = RelationshipTo('.koji.ContainerKojiBuild', 'TRIGGERED')
+    successful_koji_builds = RelationshipTo('.koji.ContainerKojiBuild', 'TRIGGERED')
+    requested_builds = RelationshipTo('.FreshmakerBuild', 'TRIGGERED')
 
     @property
     def display_name(self):
         """Get intuitive (human readable) display name for the node."""
         return 'Freshmaker event {0}'.format(self.id_)
+
+
+class FreshmakerBuild(EstuaryStructuredNode):
+    """Definition of a Freshmaker build in Neo4j."""
+
+    id_ = UniqueIdProperty(db_property='id')
+    build_id = IntegerProperty()
+    dep_on = StringProperty()
+    name = StringProperty()
+    original_nvr = StringProperty()
+    rebuilt_nvr = StringProperty()
+    state = IntegerProperty()
+    state_name = StringProperty()
+    state_reason = StringProperty()
+    time_completed = DateTimeProperty()
+    time_submitted = DateTimeProperty()
+    type_ = IntegerProperty(db_property='type')
+    type_name = StringProperty()
+    url = StringProperty()
+    koji_builds = RelationshipTo('.koji.ContainerKojiBuild', 'TRIGGERED_BY', cardinality=ZeroOrOne)
+    event = RelationshipFrom('.FreshmakerEvent', 'TRIGGERED')
+
+    @property
+    def display_name(self):
+        """Get intuitive (human readable) display name for the node."""
+        return 'Freshmaker build {0}'.format(self.id_)

estuary/utils/story.py

@@ -356,7 +356,7 @@ def story_flow(self, label):
             return {
                 'uid_name': FreshmakerEvent.id_.db_property or FreshmakerEvent.id.name,
                 'forward_relationship': '{0}>'.format(
-                    FreshmakerEvent.triggered_container_builds.definition['relation_type']),
+                    FreshmakerEvent.successful_koji_builds.definition['relation_type']),
                 'forward_relationship_display': 'triggered by',
                 'forward_label': ContainerKojiBuild.__label__,
                 'forward_label_display': 'container build',

scrapers/freshmaker.py

@@ -6,11 +6,12 @@
 import neomodel
 
 from scrapers.base import BaseScraper
-from estuary.models.freshmaker import FreshmakerEvent
+from estuary.models.freshmaker import FreshmakerEvent, FreshmakerBuild
 from estuary.models.errata import Advisory
 from estuary.models.koji import ContainerKojiBuild, KojiBuild
 from scrapers.utils import retry_session
 from estuary import log
+from estuary.utils.general import timestamp_to_datetime
 
 
 class FreshmakerScraper(BaseScraper):
@@ -72,6 +73,29 @@ def query_api_and_update_neo4j(self):
                     # To handle a faulty container build in Freshmaker
                     if not build_dict['build_id'] or int(build_dict['build_id']) < 0:
                         continue
+                    log.debug('Creating FreshmakerBuild {0}'.format(build_dict['build_id']))
+                    fb_params = dict(
+                        id_=build_dict['id'],
+                        build_dict_id=build_dict['build_dict_id'],
+                        dep_on=build_dict['dep_on'],
+                        event_id=build_dict['event_id'],
+                        name=build_dict['name'],
+                        original_nvr=build_dict['original_nvr'],
+                        rebuilt_nvr=build_dict['rebuilt_nvr'],
+                        state=build_dict['state'],
+                        state_name=build_dict['state_name'],
+                        state_reason=build_dict['state_reason'],
+                        time_submitted=timestamp_to_datetime(build_dict['time_submitted']),
+                        type_=build_dict['type'],
+                        type_name=build_dict['type_name'],
+                        url=build_dict['url']
+                    )
+                    if build_dict['time_completed']:
+                        fb_params['time_completed'] = timestamp_to_datetime(
+                            build_dict['time_completed'])
+                    fb = FreshmakerBuild.create_or_update(fb_params)[0]
+                    fb.event.connect(event)
+                    event.requested_builds.connect(fb)
 
                     # The build ID obtained from Freshmaker API is actually a Koji task ID
                     task_result = self.get_koji_task_result(build_dict['build_id'])
@@ -107,7 +131,7 @@ def query_api_and_update_neo4j(self):
                         build.add_label(ContainerKojiBuild.__label__)
                         build = ContainerKojiBuild.create_or_update(build_params)[0]
 
-                    event.triggered_container_builds.connect(build)
+                    event.successful_koji_builds.connect(build)
 
             if rv_json['meta'].get('next'):
                 fm_url = rv_json['meta']['next']

tests/api/test_all_stories.py

@@ -887,8 +887,8 @@ def test_all_stories(client, resource, uid, expected):
     commit.koji_builds.connect(build)
     build.advisories.connect(advisory)
     fm_event.triggered_by_advisory.connect(advisory)
-    fm_event.triggered_container_builds.connect(cb)
-    fm_event.triggered_container_builds.connect(cb_two)
+    fm_event.successful_koji_builds.connect(cb)
+    fm_event.successful_koji_builds.connect(cb_two)
 
     # Unique partial stories
     commit_two.resolved_bugs.connect(bug_two)

tests/api/test_general.py

@@ -116,6 +116,6 @@ def test_get_on_model_wo_uid(client, resource):
     assert rv.status_code == 400
     invalid_msg = ('The requested resource "{0}" is invalid. Choose from the following: '
                    'advisory, bugzillabug, containeradvisory, containerkojibuild, distgitcommit, '
-                   'freshmakerevent, kojibuild, kojitag, modulekojibuild, and user.'
-                   .format(resource))
+                   'freshmakerevent, freshmakerbuild, kojibuild, kojitag, modulekojibuild, and '
+                   'user.'.format(resource))
     assert json.loads(rv.data.decode('utf-8')) == {'message': invalid_msg, 'status': 400}

tests/api/test_get_resource.py

@@ -11,7 +11,7 @@
 from estuary.models.distgit import DistGitCommit, DistGitBranch, DistGitRepo
 from estuary.models.errata import Advisory
 from estuary.models.koji import KojiBuild, KojiTag, ContainerKojiBuild
-from estuary.models.freshmaker import FreshmakerEvent
+from estuary.models.freshmaker import FreshmakerEvent, FreshmakerBuild
 
 
 @pytest.mark.parametrize('resource,uid,expected', [
@@ -313,6 +313,22 @@
         'event_type_id': 8,
         'id': '1180',
         'message_id': 'ID:messaging-devops-broker01.test',
+        'requested_builds': [{
+            'build_id': 15639305,
+            'dep_on': 'jboss-eap-7-eap70-openshift-docker',
+            'id': '398',
+            'name': 'metrics-hawkular-metrics-docker',
+            'original_nvr': 'metrics-hawkular-metrics-docker-v3.7.23-10',
+            'rebuilt_nvr': 'metrics-hawkular-metrics-docker-v3.7.23-10.1522094767',
+            'state': 1,
+            'state_name': 'DONE',
+            'state_reason': 'Built successfully.',
+            'time_completed': '2017-04-02T19:39:06+00:00',
+            'time_submitted': '2017-04-02T19:39:06+00:00',
+            'type': 1,
+            'type_name': 'IMAGE',
+            'url': '/api/1/builds/398'
+        }],
         'state': 2,
         'state_name': 'COMPLETE',
         'state_reason': 'All container images have been rebuilt',
@@ -335,7 +351,7 @@
             'synopsis':'cifs-utils bug fix update',
             'update_date':'2017-08-01T07:16:00+00:00'
         },
-        'triggered_container_builds': [{
+        'successful_koji_builds': [{
             'completion_time': '2017-04-02T19:39:06+00:00',
             'creation_time': '2017-04-02T19:39:06+00:00',
             'epoch': '0',
@@ -494,6 +510,22 @@ def test_get_resources(client, resource, uid, expected):
         'state_name': 'COMPLETE',
         'state_reason': 'All container images have been rebuilt'
     })[0]
+    fm_build = FreshmakerBuild.get_or_create({
+        'id_': 398,
+        'build_id': 15639305,
+        'dep_on': "jboss-eap-7-eap70-openshift-docker",
+        'name': "metrics-hawkular-metrics-docker",
+        'original_nvr': "metrics-hawkular-metrics-docker-v3.7.23-10",
+        'rebuilt_nvr': "metrics-hawkular-metrics-docker-v3.7.23-10.1522094767",
+        'state': 1,
+        'state_name': "DONE",
+        'state_reason': "Built successfully.",
+        'time_completed': datetime(2017, 4, 2, 19, 39, 6),
+        'time_submitted': datetime(2017, 4, 2, 19, 39, 6),
+        'type_': 1,
+        'type_name': "IMAGE",
+        'url': "/api/1/builds/398"
+    })[0]
     cb = ContainerKojiBuild.get_or_create({
         'completion_time': datetime(2017, 4, 2, 19, 39, 6),
         'creation_time': datetime(2017, 4, 2, 19, 39, 6),
@@ -540,10 +572,11 @@ def test_get_resources(client, resource, uid, expected):
 
     if resource == 'freshmakerevent':
         fm_event.triggered_by_advisory.connect(advisory)
-        fm_event.triggered_container_builds.connect(cb)
+        fm_event.successful_koji_builds.connect(cb)
+        fm_event.requested_builds.connect(fm_build)
 
     if resource == 'containerbuild':
-        fm_event.triggered_container_builds.connect(cb)
+        fm_event.successful_koji_builds.connect(cb)
 
     rv = client.get('/api/v1/{0}/{1}'.format(resource, uid))
     assert rv.status_code == 200

tests/api/test_relationships.py

@@ -11,7 +11,7 @@
 
 
 @pytest.mark.parametrize('resource,uid,relationship,expected', [
-    ('freshmakerevent', '1180', 'triggered_container_builds', {
+    ('freshmakerevent', '1180', 'successful_koji_builds', {
         'data': [
             {
                 'advisories': [
@@ -111,7 +111,7 @@
             }
         ],
         'meta': {
-            'description': 'triggered container builds of Freshmaker event 1180'
+            'description': 'successful koji builds of Freshmaker event 1180'
         }
     })
 ])
@@ -159,9 +159,9 @@ def test_one_to_many_node_relationships(client, resource, uid, relationship, exp
         'version': '1.7.6'
     })[0]
 
-    fm_event.triggered_container_builds.connect(cb)
-    fm_event.triggered_container_builds.connect(cb_two)
-    fm_event.triggered_container_builds.connect(cb_three)
+    fm_event.successful_koji_builds.connect(cb)
+    fm_event.successful_koji_builds.connect(cb_two)
+    fm_event.successful_koji_builds.connect(cb_three)
 
     rv = client.get('/api/v1/relationships/{0}/{1}/{2}'.format(resource, uid, relationship))
     assert rv.status_code == 200
@@ -190,7 +190,7 @@ def test_one_to_many_node_relationships_failed(client):
         'version': '1.7.4'
     })[0]
 
-    fm_event.triggered_container_builds.connect(cb)
+    fm_event.successful_koji_builds.connect(cb)
 
     expected = {
         'message': 'Please provide a valid relationship name for freshmakerevent with uid 1180',

tests/api/test_siblings.py

@@ -438,8 +438,8 @@ def test_node_siblings(client, resource, uid, backward_rel, expected):
     build.advisories.connect(advisory)
     build_two.advisories.connect(advisory)
     fm_event.triggered_by_advisory.connect(advisory)
-    fm_event.triggered_container_builds.connect(cb)
-    fm_event.triggered_container_builds.connect(cb_two)
+    fm_event.successful_koji_builds.connect(cb)
+    fm_event.successful_koji_builds.connect(cb_two)
     ca.attached_builds.connect(cb)
     ca.attached_builds.connect(cb_two)
 
@@ -755,8 +755,8 @@ def test_module_story_node_siblings(client, resource, uid, backward_rel, expecte
     build.advisories.connect(advisory)
     build_two.advisories.connect(advisory)
     fm_event.triggered_by_advisory.connect(advisory)
-    fm_event.triggered_container_builds.connect(cb)
-    fm_event.triggered_container_builds.connect(cb_two)
+    fm_event.successful_koji_builds.connect(cb)
+    fm_event.successful_koji_builds.connect(cb_two)
     ca.attached_builds.connect(cb)
     ca.attached_builds.connect(cb_two)
     module_build.components.connect(build)