Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Use a large tg pagination value to really get all of the user's packages... #33

Merged
merged 7 commits into from

2 participants

@ralphbean
Owner

No description provided.

@pypingou
Owner

A good test for this is spot or psabata or ppisar, the own the most packages. Is 10.000 enough for them?

@pypingou
Owner

Note: pkgdb2 will likely require a similar change, I had to add pagination for the same users mentioned above :)

(and I'm thinking about removing the limit argument to prevent too easy ddos of the server)

@ralphbean
Owner

(and I'm thinking about removing the limit argument to prevent too easy ddos of the server)

Yeah, that sounds good. I guess, when the API is sorted out, let me know and I"ll update this and fmn.rules

@ralphbean
Owner

A good test for this is spot or psabata or ppisar, the own the most packages. Is 10.000 enough for them?

Yeah, I think its enough. :)

@pypingou
Owner
@pypingou
Owner
fedbadges/utils.py
@@ -143,16 +143,33 @@ def _getter(username):
def _get_pkgdb2_packages_for(config, username):
log.debug("Requesting pkgdb2 packages for user %r" % username)
- req = requests.get('{0}/packager/acl/{1}'.format(
- config['fedbadges.rules.utils.pkgdb_url'], username))
- if not req.status_code == 200:
- return set()
- data = json.loads(req.text)
+
+ def _get_page(page):
+ req = requests.get('{0}/packager/acl/{1}'.format(
+ config['fedbadges.rules.utils.pkgdb_url'], username))
+
+ if not req.status_code == 200:
+ return set()
+
+ return json.loads(req.text)
@pypingou Owner
pypingou added a note

return req.json() ?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
fedbadges/utils.py
@@ -143,16 +143,33 @@ def _getter(username):
def _get_pkgdb2_packages_for(config, username):
log.debug("Requesting pkgdb2 packages for user %r" % username)
- req = requests.get('{0}/packager/acl/{1}'.format(
- config['fedbadges.rules.utils.pkgdb_url'], username))
- if not req.status_code == 200:
- return set()
- data = json.loads(req.text)
+
+ def _get_page(page):
+ req = requests.get('{0}/packager/acl/{1}'.format(
+ config['fedbadges.rules.utils.pkgdb_url'], username))
@pypingou Owner
pypingou added a note

the page argument is not being used :)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
fedbadges/utils.py
((13 lines not shown))
if not req.status_code == 200:
return set()
+
data = json.loads(req.text)
@pypingou Owner
pypingou added a note

req.json() should work here as well if you like

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
@pypingou
Owner

One last comment but only aesthetic, :+1: for me

@ralphbean ralphbean merged commit 0709c02 into develop
@ralphbean ralphbean deleted the feature/tg-pagination branch
@ralphbean ralphbean referenced this pull request from a commit in fedora-infra/fmn.rules
@ralphbean ralphbean Copy over pkgdb pagination fixes... a872277
@ralphbean ralphbean referenced this pull request in fedora-infra/fmn.rules
Merged

Copy over pkgdb pagination fixes... #6

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
This page is out of date. Refresh to see the latest.
Showing with 46 additions and 21 deletions.
  1. +10 −8 fedbadges/rules.py
  2. +36 −13 fedbadges/utils.py
View
18 fedbadges/rules.py
@@ -178,16 +178,18 @@ def matches(self, msg):
# Limit awardees to only those who do not already have this badge.
# Do this only if we have an active connection to the Tahrir DB.
if self.tahrir:
- awardees = frozenset([user for user in awardees
- if not self.tahrir.assertion_exists(
- self.badge_id, "%s@fedoraproject.org" % user
- )])
+ awardees = frozenset([
+ user for user in awardees
+ if not self.tahrir.assertion_exists(
+ self.badge_id, "%s@fedoraproject.org" % user
+ )])
# Also, exclude any potential awardees who have opted out.
- awardees = frozenset([user for user in awardees
- if not self.tahrir.person_opted_out(
- "%s@fedoraproject.org" % user
- )])
+ awardees = frozenset([
+ user for user in awardees
+ if not self.tahrir.person_opted_out(
+ "%s@fedoraproject.org" % user
+ )])
# If no-one would get the badge at this point, then no reason to waste
# time doing any further checks. No need to query datanommer.
View
49 fedbadges/utils.py
@@ -1,6 +1,5 @@
""" Utilities for fedbadges that don't quite fit anywhere else. """
-import json
import types
import logging
@@ -143,16 +142,35 @@ def _getter(username):
def _get_pkgdb2_packages_for(config, username):
log.debug("Requesting pkgdb2 packages for user %r" % username)
- req = requests.get('{0}/packager/acl/{1}'.format(
- config['fedbadges.rules.utils.pkgdb_url'], username))
- if not req.status_code == 200:
- return set()
- data = json.loads(req.text)
+
+ def _get_page(page):
+ req = requests.get('{0}/packager/acl/{1}'.format(
+ config['fedbadges.rules.utils.pkgdb_url'], username),
+ params=dict(page=page),
+ )
+
+ if not req.status_code == 200:
+ return set()
+
+ return req.json()
+
+ # We have to request the first page of data to figure out the total number
+ data = _get_page(1)
+ pages = data['page_total']
+
packages = set()
- for pkgacl in data['acls']:
- if pkgacl['status'] != 'Approved':
- continue
- packages.add(pkgacl['packagelist']['package']['name'])
+ for i in range(1, pages + 1):
+
+ # Avoid requesting the data twice the first time around
+ if i != 1:
+ data = _get_pages(i)
+
+ for pkgacl in data['acls']:
+ if pkgacl['status'] != 'Approved':
+ continue
+
+ packages.add(pkgacl['packagelist']['package']['name'])
+
log.debug("done talking with pkgdb2 for now.")
return packages
@@ -160,12 +178,17 @@ def _get_pkgdb2_packages_for(config, username):
# TODO -- delete this once pkgdb2 goes live.
def _get_pkgdb1_packages_for(config, username):
log.debug("Requesting pkgdb1 packages for user %r" % username)
+
pkgdb1_base_url = config['fedbadges.rules.utils.pkgdb_url']
- req = requests.get('{0}/users/packages/{1}?tg_format=json'.format(
- pkgdb1_base_url, username))
+ query_string = "tg_format=json&pkgs_tgp_limit=10000"
+
+ req = requests.get('{0}/users/packages/{1}?{2}'.format(
+ pkgdb1_base_url, username, query_string))
+
if not req.status_code == 200:
return set()
- data = json.loads(req.text)
+
+ data = req.json()
packages = set([pkg['name'] for pkg in data['pkgs']])
log.debug("done talking with pkgdb1 for now.")
return packages
Something went wrong with that request. Please try again.