Permalink
Browse files

Merge remote-tracking branch 'packaginator/master'

  • Loading branch information...
2 parents 6a4a8e3 + 225ea31 commit c58460fa312b8bf48cc2416b874856278674bf30 @FlaviuSim FlaviuSim committed Mar 16, 2011
View
@@ -16,6 +16,7 @@ Direct Contributors
* Andrii Kurinny
* Brian Ball
* Bryan Weingarten
+* Chris Adams
* Eric Spunagle
* Flaviu Simihaian
* Gisle Aas (member of the Repo Men)
View
@@ -44,6 +44,9 @@ def fetch_releases(package_name, include_hidden=True):
release_data.license = classifier.replace('License ::', '')
release_data.license = release_data.license.replace('OSI Approved :: ', '')
break
-
+
+ if release_data.license and len(release_data.license) > 100:
+ release_data.license = "Other (see http://pypi.python.org/pypi/%s)" % package_name
+
releases.append(release_data)
return releases
@@ -9,7 +9,7 @@
from .base_handler import BaseHandler
-API_TARGET = "https://api.bitbucket.org/1.0/repositories/"
+API_TARGET = "https://api.bitbucket.org/1.0/repositories"
descendants_re = re.compile(r"Forks/Queues \((?P<descendants>\d+)\)")
@@ -24,7 +24,7 @@ def _get_bitbucket_commits(self, package):
repo_name = package.repo_name()
if repo_name.endswith("/"):
repo_name = repo_name[0:-1]
- target = "https://api.bitbucket.org/1.0/repositories/%s/changesets/?limit=50" % repo_name
+ target = "%s/%s/changesets/?limit=50" % (API_TARGET, repo_name)
page = urlopen(target).read()
try:
data = json.loads(page)
@@ -41,7 +41,7 @@ def fetch_commits(self, package):
def fetch_metadata(self, package):
# prep the target name
repo_name = package.repo_name()
- target = API_TARGET + repo_name
+ target = API_TARGET + "/" + repo_name
if not target.endswith("/"):
target += "/"
@@ -1,90 +1,74 @@
import re
-import xmlrpclib
from urllib import urlopen
try:
import simplejson as json
except ImportError:
import json
-
-project_name1_RE = re.compile(r'projects/([^/]+)/?$')
-project_name2_RE = re.compile(r'^http://([^/]+).sourceforge')
-
-def _sourceforge_name_from_pypi_home_page(home_page):
-
- name1 = project_name1_RE.search(home_page)
- if name1:
- name1 = name1.group(1)
-
- name2 = project_name2_RE.search(home_page)
- if name2:
- name2 = name2.group(1)
- if name2 == 'www':
- name2 = None
-
- return name1 or name2
-
-
-def _get_sourceforge_project_data(sourceforge_project_name):
- if sourceforge_project_name == None:
- return None
-
- project_json_path = 'http://sourceforge.net/api/project/name/%s/json/' % sourceforge_project_name
- # open the target and read the content
- response = urlopen(project_json_path)
- response = response.read()
-
- # dejsonify the results
- try:
- project_data = json.loads(response)['Project']
- except KeyError: # project does not exist
- project_data = None
- except ValueError: # likely invalid chars in json file
- project_data = None
-
- return project_data
-
-
-def _get_sourceforge_repo_url(sourceforge_package_data):
- # TODO: add support for hg and git.
- if 'SVNRepository' in sourceforge_package_data:
- return sourceforge_package_data['SVNRepository'].get('location', '')
- elif 'CVSRepository' in sourceforge_package_data:
- return sourceforge_package_data['SVNRepository'].get('anon-root', '')
- else:
- return ''
-
-
-def _get_sourceforge_participants(sourceforge_package_data):
- maintainers = [maintainer['name'] for maintainer in sourceforge_package_data['maintainers']]
- developers = [developer['name'] for developer in sourceforge_package_data['developers']]
- participants = maintainers + developers
- return ','.join(participants)
-
-
-def fetch_metadata(package):
- sourceforge_project_name = _sourceforge_name_from_pypi_home_page(package.pypi_home_page)
-
- # dejsonify the results
- try:
- sf_package_data = _get_sourceforge_project_data(sourceforge_project_name)
- except json.decoder.JSONDecodeError:
- message = "%s had a JSONDecodeError while loading %s" % (package.title,
- package_json_path)
- warn(message)
+from .base_handler import BaseHandler
+
+API_TARGET = "https://sourceforge.net/api"
+
+class SourceforgeError(Exception):
+ """An error occurred when making a request to the Sourceforge API"""
+
+class SourceforgeHandler(BaseHandler):
+ """
+ The Sourceforge API has some tricky stuff in it - some sections are fed
+ via xml/rss, some are via json. As of 03/16/2011, the xml API is the most
+ up-to-date, but a bug has been opened to fix the json side. This API is
+ on hold until it is fixed.
+ """
+
+ title = "Sourceforge"
+ url = "https://sourceforge.net"
+ repo_regex = r'https://sourceforge.com/[\w\-\_]+/([\w\-\_]+)/{0,1}'
+ slug_regex = r'https://sourceforge.com/[\w\-\_]+/([\w\-\_]+)/{0,1}'
+
+ def fetch_metadata(self, package):
+ sourceforge = '';
+
+ repo_name = package.repo_name()
+ target = API_TARGET + "/projects/name/" + repo_name
+ if not target.endswith("/"):
+ target += "/"
+
+ # sourceforge project API requires ending with /doap/
+ target += "json/"
+
+ # open the target and read the content
+ response = urlopen(target)
+ response_text = response.read()
+
+ # dejson the results
+ try:
+ data = json.loads(response_text)
+ except jason.decoder.JSONDecodeError:
+ raise SourceforgeError("unexpected response from sourceforge.net %d: %r" % (
+ response.status, response_text))
+
+ # sourceforge has both developers and maintainers in a list
+ participants = data.get("developers").append(data.get("maintainers"))
+ package.participants = [p['name'] for p in participants]
+
+ package.repo_description = data.get("description")
+
+ project_name = _name_from_pypi_home_page(package.pypi_home_page)
+ # dejsonify the results
+ try:
+ sf_package_data = _get_project_data(project_name)
+ except json.decoder.JSONDecodeError:
+ message = "%s had a JSONDecodeError while loading %s" % (package.title,
+ package_json_path)
+ warn(message)
+ return package
+ package.repo_watchers = len(sf_package_data.get('maintainers', [])) + len(sf_package_data.get('developers', []))
+ package.repo_description = sf_package_data.get('description', '')
+ # TODO - remove the line below and use repo_url as your foundation
+ package.repo_url = _get_repo_url(sf_package_data)
+ package.repo_forks = None
+
return package
-
- package.repo_watchers = len(sf_package_data.get('maintainers', [])) + len(sf_package_data.get('developers', []))
- package.repo_description = sf_package_data.get('description', '')
- # TODO - remove the line below and use repo_url as your foundation
- package.repo_url = _get_sourceforge_repo_url(sf_package_data)
- package.repo_forks = None
- package.participants = _get_sourceforge_participants(sf_package_data)
-
- return package
-
-class SourceforgeHandler(object):
- pass
-
-repo_handler = SourceforgeHandler()
+
+repo_handler = SourceforgeHandler()
View
@@ -0,0 +1,9 @@
+from celery.decorators import task
+
+from packaginator.apps.package.models import Package
+
+
+@task()
+def fetch_package_metadata(pk):
+ package = Package.objects.get(pk=pk)
+ package.fetch_metadata()
@@ -1,5 +1,6 @@
-from package.tests.test_pulls import *
+#from package.tests.test_pulls import *
+from package.tests.test_models import *
from package.tests.test_repos import *
-from package.tests.test_sourceforge import *
+#from package.tests.test_sourceforge import *
from package.tests.test_utils import *
from package.tests.test_views import *

Large diffs are not rendered by default.

Oops, something went wrong.
@@ -0,0 +1 @@
+[{"pk": 2278, "model": "package.version", "fields": {"license": "BSD License", "package": 6, "downloads": 1904, "created": "2011-02-22 14:01:36", "number": "2.1.3", "modified": "2011-03-16 04:45:48", "hidden": false}}, {"pk": 2252, "model": "package.version", "fields": {"license": "BSD License", "package": 6, "downloads": 715, "created": "2011-02-16 15:01:38", "number": "2.1.2", "modified": "2011-03-16 04:45:48", "hidden": false}}, {"pk": 2177, "model": "package.version", "fields": {"license": "BSD License", "package": 6, "downloads": 906, "created": "2011-02-09 13:02:16", "number": "2.1.1", "modified": "2011-03-16 04:45:48", "hidden": false}}, {"pk": 2041, "model": "package.version", "fields": {"license": "BSD License", "package": 6, "downloads": 1613, "created": "2011-01-26 17:01:34", "number": "2.1.0", "modified": "2011-03-16 04:45:48", "hidden": false}}, {"pk": 1977, "model": "package.version", "fields": {"license": "BSD License", "package": 6, "downloads": 850, "created": "2011-01-14 13:01:28", "number": "2.1.0.rc3", "modified": "2011-03-16 04:45:48", "hidden": false}}, {"pk": 1913, "model": "package.version", "fields": {"license": "BSD License", "package": 6, "downloads": 726, "created": "2010-12-31 12:01:20", "number": "2.1.0.rc2", "modified": "2011-03-16 04:45:48", "hidden": false}}, {"pk": 1870, "model": "package.version", "fields": {"license": "BSD License", "package": 6, "downloads": 299, "created": "2010-12-24 09:01:10", "number": "2.1.0.rc1", "modified": "2011-03-16 04:45:48", "hidden": false}}, {"pk": 12, "model": "package.version", "fields": {"license": "BSD License", "package": 6, "downloads": 1062, "created": "2010-09-11 23:05:40", "number": "2.0.0", "modified": "2011-03-16 04:45:48", "hidden": false}}, {"pk": 11, "model": "package.version", "fields": {"license": "BSD License", "package": 6, "downloads": 212, "created": "2010-09-11 23:05:40", "number": "2.0.1", "modified": "2011-03-16 04:45:48", "hidden": false}}, {"pk": 10, "model": "package.version", "fields": {"license": "BSD License", "package": 6, "downloads": 4326, "created": "2010-09-11 23:05:40", "number": "2.0.2", "modified": "2011-03-16 04:45:48", "hidden": false}}, {"pk": 9, "model": "package.version", "fields": {"license": "BSD License", "package": 6, "downloads": 13644, "created": "2010-09-11 23:05:40", "number": "2.1.0.beta3", "modified": "2011-03-16 04:45:48", "hidden": false}}]
@@ -0,0 +1,25 @@
+from django.test import TestCase
+
+from package.models import Package, Version, versioner
+
+class VersionTests(TestCase):
+
+ fixtures = ['apps/package/tests/test_data/versioner_test_fixture.json',
+ 'apps/package/tests/test_data/versioner_versions_fixture.json']
+
+ def test_version_order(self):
+ p = Package.objects.get(slug='django-cms')
+ versions = p.version_set.by_version()
+ expected_values = [ '2.0.0',
+ '2.0.1',
+ '2.0.2',
+ '2.1.0',
+ '2.1.0.beta3',
+ '2.1.0.rc1',
+ '2.1.0.rc2',
+ '2.1.0.rc3',
+ '2.1.1',
+ '2.1.2',
+ '2.1.3']
+ returned_values = [v.number for v in versions]
+ self.assertEquals(returned_values,expected_values)
@@ -5,7 +5,7 @@
from package.repos.bitbucket import repo_handler as bitbucket_handler
from package.repos.github import repo_handler as github_handler
from package.repos.launchpad import repo_handler as launchpad_handler
-from package.repos.sourceforge import repo_handler as sourceforge_handler
+#from package.repos.sourceforge import repo_handler as sourceforge_handler
from package.models import Commit, Package
View
@@ -15,6 +15,7 @@ sys.path.insert(0, abspath(join(dirname(__file__), "../../")))
from django.conf import settings
os.environ["DJANGO_SETTINGS_MODULE"] = "djangopackages.settings"
+os.environ["CELERY_LOADER"] = "django"
sys.path.insert(0, join(settings.PROJECT_ROOT, "apps"))
View
@@ -16,6 +16,7 @@ Direct Contributors
* Andrii Kurinny
* Brian Ball
* Bryan Weingarten
+* Chris Adams
* Eric Spunagle
* Flaviu Simihaian
* Gisle Aas (member of the Repo Men)
View
@@ -2,6 +2,8 @@
Installation
============
+Do everything listed in this section to get your site up and running locally. If you run into problems, see the Troubleshooting section.
+
Pre-requisites
==============
@@ -104,9 +106,13 @@ Replace joe with your username/email::
Install Djangopackages flatblocks and flatpages
===============================================
-Packaginator makes use of several flatblocks and flatpages. To load fixtures
-that show how these are used on djangopackages.com, you can load two fixtures
-containing this information::
+Packaginator makes use of several flatblocks and flatpages.
+
+To see how the flatblocks and flatpages are used on djangopackages.com, open fixtures/flatblocks.json and fixtures/flatpages.json in a text editor. Change "Django Packages" to "Python Packages" or whatever the name of your site is.
+
+Change other parts of the text if you want (note: you can do this later via the Django admin interface under flatblocks/flatpages as well).
+
+Then, you can load the two flatblocks and flatpages fixtures::
python manage.py loaddata fixtures/flatblocks.json
python manage.py loaddata fixtures/flatpages.json
View
@@ -2,34 +2,36 @@
Introduction
=============
-Packaginator solves the problem in the Python community of being able to easily identify good apps, frameworks, and packages. Ever want to know which is the most popular or well supported Python httplib replacement, web framework, or api tool? Packaginator solves that problem for you!
+Ever want to know which is the most popular or well supported Python httplib replacement, web framework, or api tool? Packaginator solves that problem for you!
-It does this by storing information on packages fetched from public APIs provided by PyPI, Github, BitBucket, Launchpad, and SourceForge, and then provides extremely useful comparison tools for them.
+Easily identify good apps, frameworks, and packages.
+
+Packaginator stores information on fetched packages and provides easy comparison tools for them. Public APIs include PyPI, Github, BitBucket, Launchpad, and SourceForge.
The Site
--------
-The most current example is live and functional at http://www.djangopackages.com.
+A current example is live: http://www.djangopackages.com
Grids!
~~~~~~
-Grids let you compare packages to each other. A grid comes with a number of default items compared, but you can add more features in order to get a more specific comparison.
+Grids let you compare packages. A grid comes with default comparison items and you can add features to get a more specific.
-We think grids are a gigantic improvement over the traditional tagging system, because we think that grids give us a lot more specificity.
+We think comparison grids are an improvement over traditional tagging system because specificity helps make informed decisions.
Categories of Packages
~~~~~~~~~~~~~~~~~~~~~~
The fixtures provide four categories: apps, frameworks, projects, and utilities.
Google Project Hosting and Sourceforge are not fully supported!
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Not yet.
+Not yet.
-The progenitor of Packaginator, Django Packages was cooked up during Django Dash 2010. We wanted to keep the scope of our work reasonable. We'll try and include those sites in the future. We also want to include other package repo systems over time. As for what we support:
+The progenitor of Packaginator, Django Packages was cooked up during Django Dash 2010. We wanted to keep the scope of our work reasonable. We'll try to include more sites in the future. Here's what we support:
- * Packaginator does support Github and Bitbucket and Launchpad.
- * Sourceforge needs some tweaking but is otherwise it is done.
- * Google Project Hosting may not happen because of a lack of a formal API and not much desire to screen scrape their arcane browser interface.
+ * Github, Bitbucket and Launchpad.
+ * Sourceforge needs tweaking.
+ * NOT YET: Google Project Hosting. Google's lack of a formal API leaves us the option of screen-scraping their content. We're not excited about that.
View
@@ -9,10 +9,12 @@ If you're getting something like "ImportError: No module named abc", you probabl
pip install -r requirements/project.txt
-Launchpad/bzr installation problems
+ImportError related to launchpad.py
-----------------------------------
-Most likely cause is your C compiler. On Windows, make sure you have MinGW installed as per the installation instructions. On Linux, make sure you have the python-dev and gcc packages.
+Sometimes this shows up as "Caught ImportError while rendering: cannot import name ScalarValue".
+
+You're having Launchpad/bzr installation problems. Most likely cause is your C compiler. On Windows, make sure you have MinGW installed as per the installation instructions. On Linux, make sure you have the python-dev and gcc packages.
Other problems
--------------
@@ -42,3 +42,10 @@ SECRET_KEY = "CHANGE-THIS-KEY-TO-SOMETHING-ELSE"
if SECRET_KEY == "CHANGE-THIS-KEY-TO-SOMETHING-ELSE":
raise Exception('You must change your SECRET_KEY settings in '
'local_settings.py.')
+
+# See http://celeryproject.org/docs/configuration.html#task-execution-settings
+CELERY_ALWAYS_EAGER = True
+CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
+BROKER_BACKEND = "django"
+
+LOCAL_INSTALLED_APPS = ('djkombu', )
View
@@ -17,3 +17,9 @@ coverage==3.4
django-coverage==1.1.1
django-flatblocks==0.5.0
feedparser==5.0.1
+
+# Celery dependencies:
+celery==2.2.4
+kombu==1.0.4
+django-celery==2.2.4
+django-kombu==0.9.2
Oops, something went wrong.

0 comments on commit c58460f

Please sign in to comment.