This repository has been archived by the owner on Jul 25, 2018. It is now read-only.
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
37b73c8
commit 79d6d94
Showing
7 changed files
with
267 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
|
|
||
|
|
||
| {% extends "exp.html" %} | ||
|
|
||
| {% block title %}Packages lookup{% end %} | ||
|
|
||
| {% block content %} | ||
| <p> | ||
| Looks up packages on PyPI, CPAN, RubyGems, NPM, Maven Central, Clojars, | ||
| GitHub, Bitbucket, Ubuntu, Arch User Repository, FreeBSD ports and Homebrew. | ||
| </p> | ||
| <p> | ||
| Results are cached for five hours. | ||
| </p> | ||
| <form id='theform'> | ||
| <input type='text' id='query'> | ||
| <button id='go' disabled>Go!</button> | ||
| </form> | ||
| <div id='spintarget'></div> | ||
| <ul id='results'> | ||
| </ul> | ||
| {% end %} | ||
|
|
||
| {% block head %} | ||
| <script src='/assets/reqwest.min.js'></script> | ||
| <style> | ||
| @media screen and (min-width: 700px) { | ||
| #results { column-count: 2; column-gap: 1em } | ||
| } | ||
| @media screen and (min-width: 1200px) { | ||
| #results { column-count: 3 } | ||
| } | ||
| #results { list-style-type: none } | ||
| #results a { color: inherit } | ||
| .pypi { color: #c4a000 } | ||
| .rubygems { color: #a40000 } | ||
| .freebsd { color: #ef2929 } | ||
| .maven { color: #3465a4 } | ||
| .aur { color: #729fcf } | ||
| .clojars { color: #4e9a06 } | ||
| .ubuntu { color: #8f5902 } | ||
| .npm { color: #75507b } | ||
| .homebrew { color: #ce5c00 } | ||
| .github, .bitbucket { color: #555753 } | ||
| </style> | ||
| {% end %} | ||
|
|
||
| {% block js %} | ||
| domready(function() { | ||
| function $() { return document.getElementById.apply(document, arguments) } | ||
| $('go').disabled = false | ||
| $('theform').onsubmit = function() { | ||
| var spinner = FSpinner().spin($('spintarget')) | ||
| reqwest({ | ||
| url: '/pkglookup/lookup.json', data: {q: $('query').value}, | ||
| type: 'json', method: 'get', | ||
| error: function(err) { | ||
| console.log(err) | ||
| spinner.stop() | ||
| $('results').innerHTML = '<li>FAIL, see console</li>' | ||
| }, | ||
| success: function(resp) { | ||
| var acc = '' | ||
| resp.results.forEach(function(result) { | ||
| acc += '<li class='+ result.source.toLowerCase() +'>' + | ||
| '<a href="'+ result.url +'">'+ result.name +'</a> on '+ result.source +'</li>' | ||
| }) | ||
| spinner.stop() | ||
| $('results').innerHTML = acc | ||
| $('query').value = '' | ||
| } | ||
| }) | ||
| return false | ||
| } | ||
| }) | ||
| {% end %} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,162 @@ | ||
| import json | ||
| import requests | ||
| import xmlrpclib | ||
| from Queue import Queue | ||
| from threading import Thread | ||
| from config import REDIS as red | ||
| from bs4 import BeautifulSoup as BS | ||
|
|
||
| last_ubuntu = 'precise' | ||
| lim = 7 | ||
| cache_hours = 5 | ||
| [red.delete(k) for k in red.keys('PKG*')] # Delete cached stuff when reloading the app | ||
|
|
||
| query = request.qs.one('q') | ||
| cachekey = 'PKG'+query | ||
| cached = red.get(cachekey) | ||
|
|
||
| if cached: | ||
| response.body = cached | ||
| else: | ||
| global results | ||
| global queue | ||
| results = [] | ||
| queue = Queue() | ||
|
|
||
| def fetcher(fn): | ||
| def inner(): | ||
| global results | ||
| global queue | ||
| query = queue.get() | ||
| results += fn(query) | ||
| queue.task_done() | ||
| return inner | ||
|
|
||
| @fetcher | ||
| def l_pypi(query): | ||
| resp = xmlrpclib.ServerProxy('http://pypi.python.org/pypi').search({'name': query}) | ||
| return [{'name': entry['name'], 'source': 'PyPI', | ||
| 'url': 'http://pypi.python.org/pypi/' + entry['name']} | ||
| for entry in resp[:lim]] | ||
|
|
||
| @fetcher | ||
| def l_cpan(query): | ||
| resp = requests.get('http://search.cpan.org/search', | ||
| params={'mode': 'module', 'format': 'xml', 'query': query}) | ||
| if resp.status_code == 200: | ||
| return [{'name': entry.find('name').string, 'source': 'CPAN', | ||
| 'url': ''.join(entry.find('link').strings)} | ||
| for entry in BS(resp.text, 'html.parser').find_all('module')[:lim]] | ||
| # html.parser parses <link>http://an.url</link> correctly | ||
| return [] | ||
|
|
||
| @fetcher | ||
| def l_rubygems(query): | ||
| resp = requests.get('https://rubygems.org/api/v1/search.json', | ||
| params={'query': query}) | ||
| if resp.status_code == 200: | ||
| return [{'name': entry['name'], 'source': 'RubyGems', | ||
| 'url': entry['project_uri']} | ||
| for entry in json.loads(resp.text)[:lim]] | ||
| return [] | ||
|
|
||
| @fetcher | ||
| def l_npm(query): | ||
| resp = requests.get('http://search.npmjs.org/_list/search/search', | ||
| params={'startkey': '"%s"' % query, | ||
| 'endkey': '"%sZZZZZZZZZZZZZZZZZZZ"' % query, 'limit': lim}) | ||
| if resp.status_code == 200: | ||
| return [{'name': entry['key'], 'source': 'npm', | ||
| 'url': 'http://search.npmjs.org/#/' + entry['key']} | ||
| for entry in json.loads(resp.text)['rows']] | ||
| return [] | ||
|
|
||
| @fetcher | ||
| def l_maven(query): | ||
| resp = requests.get('http://search.maven.org/solrsearch/select', | ||
| params={'wt': 'json', 'rows': lim, 'q': query}) | ||
| if resp.status_code == 200: | ||
| return [{'name': entry['id'], 'source': 'Maven', | ||
| 'url': 'http://search.maven.org/#artifactdetails|%s|%s|%s|jar' | ||
| % (entry['g'], entry['a'], entry['latestVersion'])} | ||
| for entry in json.loads(resp.text)['response']['docs']] | ||
| return [] | ||
|
|
||
| @fetcher | ||
| def l_clojars(query): | ||
| resp = requests.get('http://clojars.org/search', params={'q': query}) | ||
| if resp.status_code == 200: | ||
| return [{'name': entry.find('a').string, 'source': 'Clojars', | ||
| 'url': 'http://clojars.org/' + entry.find('a').string} | ||
| for entry in BS(resp.text).find_all('li', 'search-results')[:lim]] | ||
| return [] | ||
|
|
||
| @fetcher | ||
| def l_github(query): | ||
| resp = requests.get('http://github.com/api/v2/json/repos/search/'+query) | ||
| if resp.status_code == 200: | ||
| return [{'name': entry['name'], 'source': 'GitHub', 'url': entry['url']} | ||
| for entry in json.loads(resp.text)['repositories'][:lim]] | ||
| return [] | ||
|
|
||
| @fetcher | ||
| def l_bitbucket(query): | ||
| resp = requests.get('https://api.bitbucket.org/1.0/repositories/', params={'name': query}) | ||
| if resp.status_code == 200: | ||
| return [{'name': entry['name'], 'source': 'Bitbucket', | ||
| 'url': entry['resource_uri'].replace('/api/1.0/repositories', 'https://bitbucket.org')} | ||
| for entry in json.loads(resp.text)['repositories'][:lim]] | ||
| return [] | ||
|
|
||
| @fetcher | ||
| def l_ubuntu(query): | ||
| resp = requests.get('http://packages.ubuntu.com/search', | ||
| params={'suite': last_ubuntu, 'searchon': 'names', 'keywords': query}) | ||
| if resp.status_code == 200: | ||
| return [{'name': entry.string.replace('Package ', ''), 'source': 'Ubuntu', | ||
| 'url': 'http://packages.ubuntu.com/%s/%s' % \ | ||
| (last_ubuntu, entry.string.replace('Package ', ''))} | ||
| for entry in BS(resp.text).find('div', id='psearchres').find_all('h3')[:lim]] | ||
| return [] | ||
|
|
||
| @fetcher | ||
| def l_aur(query): | ||
| resp = requests.get('https://aur.archlinux.org/rpc.php', | ||
| params={'type': 'search', 'arg': query}) | ||
| if resp.status_code == 200: | ||
| return [{'name': entry['Name'], 'source': 'AUR', | ||
| 'url': 'http://aur.archlinux.org/packages.php?ID=' + entry['ID']} | ||
| for entry in json.loads(resp.text)['results'][:lim]] | ||
| return [] | ||
|
|
||
| @fetcher | ||
| def l_freebsd(query): | ||
| resp = requests.get('http://www.freebsd.org/cgi/ports.cgi', | ||
| params={'stype': 'name', 'sektion': 'all', 'query': query}) | ||
| if resp.status_code == 200: | ||
| acc = [] | ||
| for category in BS(resp.text).find_all('dl'): | ||
| acc += [{'name': entry.find_all('a')[1].string, 'source': 'FreeBSD', | ||
| 'url': entry.find_all('a')[1]['href']} | ||
| for entry in category.find_all('dt')[:lim]] | ||
| return acc | ||
| return [] | ||
|
|
||
| @fetcher | ||
| def l_homebrew(query): | ||
| resp = requests.get('http://braumeister.org/search/'+query) | ||
| if resp.status_code == 200: | ||
| return [{'name': entry.find('a').string, 'source': 'Homebrew', | ||
| 'url': 'http://braumeister.org' + entry.find('a')['href']} | ||
| for entry in BS(resp.text).find_all('div', 'formula')[:lim]] | ||
|
|
||
| for fn in [l_pypi, l_cpan, l_rubygems, l_npm, l_maven, | ||
| l_clojars, l_github, l_bitbucket, l_ubuntu, | ||
| l_aur, l_freebsd, l_homebrew]: | ||
| Thread(target=fn).start() | ||
| queue.put(query) | ||
|
|
||
| queue.join() | ||
| response.body = json.dumps({'results': results}) | ||
| red.set(cachekey, response.body) | ||
| red.expire(cachekey, 60*60*cache_hours) |