Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Add gerritdm

  • Loading branch information...
commit 712a2bbf1c00dc92046990106b2c92c63bc4eb4c 1 parent 171b4e8
Mark McLoughlin authored April 06, 2012
6  database.py
@@ -129,6 +129,7 @@ def __init__ (self, name):
129 129
         self.added = self.removed = self.count = self.changed = 0
130 130
         self.sobs = 0
131 131
         self.bugsfixed = [ ]
  132
+        self.reviews = [ ]
132 133
         self.hackers = [ ]
133 134
 
134 135
     def AddCSet (self, patch):
@@ -147,6 +148,11 @@ def AddBug (self, bug):
147 148
         if bug.owner not in self.hackers:
148 149
             self.hackers.append (bug.owner)
149 150
 
  151
+    def AddReview (self, reviewer):
  152
+        self.reviews.append(reviewer)
  153
+        if reviewer not in self.hackers:
  154
+            self.hackers.append (reviewer)
  155
+
150 156
 Employers = { }
151 157
 
152 158
 def GetEmployer (name):
100  gerrit/parse-reviews.py
... ...
@@ -0,0 +1,100 @@
  1
+import argparse
  2
+import json
  3
+import sys
  4
+import time
  5
+
  6
+#
  7
+# List reviewers for a set of git commits
  8
+#
  9
+#  python buglist.py essex-commits.txt openstack-config/launchpad-ids.txt < gerrit.json
  10
+#
  11
+
  12
+parser = argparse.ArgumentParser(description='List reviewers in gerrit')
  13
+
  14
+parser.add_argument('commits', help='path to list of commits to consider')
  15
+parser.add_argument('usermap', help='path to username to email map')
  16
+
  17
+args = parser.parse_args()
  18
+
  19
+username_to_email_map = {}
  20
+for l in open(args.usermap, 'r'):
  21
+    (username, email) = l.split()
  22
+    username_to_email_map.setdefault(username, email)
  23
+
  24
+commits = [l.strip() for l in open(args.commits, 'r')]
  25
+
  26
+class Reviewer:
  27
+    def __init__(self, username, name, email):
  28
+        self.username = username
  29
+        self.name = name
  30
+        self.email = email if email else username_to_email_map.get(self.username)
  31
+
  32
+    @classmethod
  33
+    def parse(cls, r):
  34
+        return cls(r.get('username'), r.get('name'), r.get('email'))
  35
+
  36
+class Approval:
  37
+    CodeReviewed, Approved, Submitted, Verified = range(4)
  38
+
  39
+    type_map = {
  40
+        'CRVW': CodeReviewed,
  41
+        'APRV': Approved,
  42
+        'SUBM': Submitted,
  43
+        'VRIF': Verified,
  44
+        }
  45
+
  46
+    def __init__(self, type, value, date, by):
  47
+        self.type = type
  48
+        self.value = value
  49
+        self.date = date
  50
+        self.by = by
  51
+
  52
+    @classmethod
  53
+    def parse(cls, a):
  54
+        return cls(cls.type_map[a['type']],
  55
+                   int(a['value']),
  56
+                   time.gmtime(int(a['grantedOn'])),
  57
+                   Reviewer.parse(a['by']))
  58
+
  59
+class PatchSet:
  60
+    def __init__(self, revision, approvals):
  61
+        self.revision = revision
  62
+        self.approvals = approvals
  63
+
  64
+    @classmethod
  65
+    def parse(cls, ps):
  66
+        return cls(ps['revision'],
  67
+                   [Approval.parse(a) for a in ps.get('approvals', [])])
  68
+
  69
+class Review:
  70
+    def __init__(self, id, patchsets):
  71
+        self.id = id
  72
+        self.patchsets = patchsets
  73
+
  74
+    @classmethod
  75
+    def parse(cls, r):
  76
+        return cls(r['id'],
  77
+                   [PatchSet.parse(ps) for ps in r['patchSets']])
  78
+
  79
+reviews = [Review.parse(json.loads(l)) for l in sys.stdin if not 'runTimeMilliseconds' in l]
  80
+
  81
+def reviewers(review):
  82
+    ret = {}
  83
+    for ps in r.patchsets:
  84
+        for a in ps.approvals:
  85
+            if a.type == Approval.CodeReviewed and a.value:
  86
+                ret.setdefault(a.by.username, (a.by, a.date))
  87
+    return ret.values()
  88
+
  89
+def interesting(review):
  90
+    for ps in r.patchsets:
  91
+        if ps.revision in commits:
  92
+            return True
  93
+    return False
  94
+
  95
+for r in reviews:
  96
+    if not interesting(r):
  97
+        continue
  98
+    for reviewer, date in reviewers(r):
  99
+        if reviewer.email:
  100
+            print time.strftime('%Y-%m-%d', date), reviewer.username, reviewer.email
119  gerritdm
... ...
@@ -0,0 +1,119 @@
  1
+#!/usr/bin/pypy
  2
+#-*- coding:utf-8 -*-
  3
+#
  4
+
  5
+#
  6
+# This code is part of the LWN git data miner.
  7
+#
  8
+# Copyright 2007-11 Eklektix, Inc.
  9
+# Copyright 2007-11 Jonathan Corbet <corbet@lwn.net>
  10
+# Copyright 2011 Germán Póo-Caamaño <gpoo@gnome.org>
  11
+#
  12
+# This file may be distributed under the terms of the GNU General
  13
+# Public License, version 2.
  14
+
  15
+
  16
+import database, ConfigFile, reports
  17
+import getopt, datetime
  18
+import sys
  19
+
  20
+Today = datetime.date.today()
  21
+
  22
+#
  23
+# Control options.
  24
+#
  25
+MapUnknown = 0
  26
+DevReports = 1
  27
+DumpDB = 0
  28
+CFName = 'gitdm.config'
  29
+DirName = ''
  30
+
  31
+#
  32
+# Options:
  33
+#
  34
+# -b dir	Specify the base directory to fetch the configuration files
  35
+# -c cfile	Specify a configuration file
  36
+# -d		Output individual developer stats
  37
+# -h hfile	HTML output to hfile
  38
+# -l count	Maximum length for output lists
  39
+# -o file	File for text output
  40
+# -p prefix Prefix for CSV output
  41
+# -s		Ignore author SOB lines
  42
+# -u		Map unknown employers to '(Unknown)'
  43
+# -z		Dump out the hacker database at completion
  44
+
  45
+def ParseOpts ():
  46
+    global MapUnknown, DevReports
  47
+    global DumpDB
  48
+    global CFName, DirName, Aggregate
  49
+
  50
+    opts, rest = getopt.getopt (sys.argv[1:], 'b:dc:h:l:o:uz')
  51
+    for opt in opts:
  52
+        if opt[0] == '-b':
  53
+            DirName = opt[1]
  54
+        elif opt[0] == '-c':
  55
+            CFName = opt[1]
  56
+        elif opt[0] == '-d':
  57
+            DevReports = 0
  58
+        elif opt[0] == '-h':
  59
+            reports.SetHTMLOutput (open (opt[1], 'w'))
  60
+        elif opt[0] == '-l':
  61
+            reports.SetMaxList (int (opt[1]))
  62
+        elif opt[0] == '-o':
  63
+            reports.SetOutput (open (opt[1], 'w'))
  64
+        elif opt[0] == '-u':
  65
+            MapUnknown = 1
  66
+        elif opt[0] == '-z':
  67
+            DumpDB = 1
  68
+
  69
+def LookupStoreHacker (date, name, email):
  70
+    email = database.RemapEmail (email)
  71
+    h = database.LookupEmail (email)
  72
+    if h: # already there
  73
+        return date, h
  74
+    elist = database.LookupEmployer (email, MapUnknown)
  75
+    h = database.LookupName (name)
  76
+    if h: # new email
  77
+        h.addemail (email, elist)
  78
+        return date, h
  79
+    return date, database.StoreHacker(name, elist, email)
  80
+
  81
+#
  82
+# Here starts the real program.
  83
+#
  84
+ParseOpts ()
  85
+
  86
+#
  87
+# Read the config files.
  88
+#
  89
+ConfigFile.ConfigFile (CFName, DirName)
  90
+
  91
+reviews = [LookupStoreHacker(*l.split()[:3]) for l in sys.stdin]
  92
+
  93
+for date, reviewer in reviews:
  94
+    reviewer.addreview(reviewer)
  95
+    empl = reviewer.emailemployer(reviewer.email[0], ConfigFile.ParseDate(date))
  96
+    empl.AddReview(reviewer)
  97
+
  98
+if DumpDB:
  99
+    database.DumpDB ()
  100
+database.MixVirtuals ()
  101
+
  102
+#
  103
+# Say something
  104
+#
  105
+hlist = database.AllHackers ()
  106
+elist = database.AllEmployers ()
  107
+ndev = nempl = 0
  108
+for h in hlist:
  109
+    if len(h.reviews) > 0:
  110
+        ndev += 1
  111
+for e in elist:
  112
+    if len(e.reviews) > 0:
  113
+        nempl += 1
  114
+reports.Write ('Processed %d review from %d developers\n' % (len(reviews), ndev))
  115
+reports.Write ('%d employers found\n' % (nempl))
  116
+
  117
+if DevReports:
  118
+    reports.DevReviews (hlist, len(reviews))
  119
+reports.EmplReviews (elist, len(reviews))
43  openstack-config/README
@@ -65,3 +65,46 @@ Launchpad API docs are here:
65 65
 
66 66
   https://launchpad.net/+apidoc/1.0.html
67 67
   https://help.launchpad.net/API/launchpadlib
  68
+
  69
+== Gerrit ==
  70
+
  71
+First, generate a list of Change-Ids:
  72
+
  73
+  $> grep -v '^#' openstack-config/essex | \
  74
+      while read project revisions; do \
  75
+       (cd ~/git/openstack/$project; \
  76
+        git fetch origin 2>/dev/null; \
  77
+        git log $revisions); \
  78
+      done | \
  79
+        awk '/^    Change-Id: / { print $2 }' | \
  80
+        split -l 100 -d - essex-change-ids-
  81
+
  82
+The output is split across files of 100 lines each because gerrit's
  83
+query will only return 500 results at a time.
  84
+
  85
+Now, we generate a raw json query result:
  86
+
  87
+  $> for f in essex-change-ids-??; do
  88
+        ssh -p 29418 review.openstack.org \
  89
+            gerrit query --all-approvals --format=json \
  90
+                $(awk -v ORS=" OR "  '{print}' $f | sed 's/ OR $//') ; \
  91
+      done > essex-reviews.txt
  92
+
  93
+Next, generate a list of commits:
  94
+
  95
+  $> grep -v '^#' openstack-config/essex | \
  96
+      while read project revisions; do \
  97
+       (cd ~/git/openstack/$project; \
  98
+        git fetch origin 2>/dev/null; \
  99
+        git log --pretty=format:%H $revisions); \
  100
+      done > essex-commits.txt
  101
+
  102
+Now parse the json into a list of reviewers:
  103
+
  104
+  $> python gerrit/parse-reviews.py \
  105
+       essex-commits.txt openstack-config/launchpad-ids.txt \
  106
+       < essex-reviews.txt  > essex-reviewers.txt
  107
+
  108
+Finally, generate the stats with:
  109
+
  110
+  $> python ./gerritdm -l 20 < essex-reviewers.txt
25  reports.py
@@ -232,6 +232,25 @@ def ReportByRevs (hlist):
232 232
             break
233 233
     EndReport ()
234 234
 
  235
+def CompareRevsEmpl (e1, e2):
  236
+    return len (e2.reviews) - len (e1.reviews)
  237
+
  238
+def ReportByRevsEmpl (elist):
  239
+    elist.sort (CompareRevsEmpl)
  240
+    totalrevs = 0
  241
+    for e in elist:
  242
+        totalrevs += len (e.reviews)
  243
+    count = 0
  244
+    BeginReport ('Top reviewers by employer (total %d)' % totalrevs)
  245
+    for e in elist:
  246
+        scount = len (e.reviews)
  247
+        if scount > 0:
  248
+            ReportLine (e.name, scount, (scount*100.0)/totalrevs)
  249
+        count += 1
  250
+        if count >= ListCount:
  251
+            break
  252
+    EndReport ()
  253
+
235 254
 #
236 255
 # tester reporting.
237 256
 #
@@ -377,6 +396,12 @@ def DevBugReports (hlist, totalbugs):
377 396
 def EmplBugReports (elist, totalbugs):
378 397
     ReportByBCEmpl (elist, totalbugs)
379 398
 
  399
+def DevReviews (hlist, totalreviews):
  400
+    ReportByRevs (hlist)
  401
+
  402
+def EmplReviews (elist, totalreviews):
  403
+    ReportByRevsEmpl (elist)
  404
+
380 405
 def ReportByFileType (hacker_list):
381 406
     total = {}
382 407
     total_by_hacker = {}

0 notes on commit 712a2bb

Please sign in to comment.
Something went wrong with that request. Please try again.