Skip to content
Permalink
Browse files
Add views for downloading data
Signed-off-by: Chris Warrick <kwpolska@gmail.com>
  • Loading branch information
Kwpolska committed Sep 22, 2018
1 parent debea54 commit 29e48f3e2d2226c84b224e315778c0faee6fbd5a
Showing with 93 additions and 7 deletions.
  1. +6 −0 sites/urls.py
  2. +72 −7 sites/views.py
  3. +15 −0 templates/downloads.html
@@ -1,6 +1,7 @@
from django.conf.urls import url
from . import views

app_name = 'sites'
urlpatterns = [
url(r'^$', views.index, name='index'),
url(r'^add/?$', views.add, name='add'),
@@ -12,4 +13,9 @@
url(r'^lang/?$', views.langlist, name='langlist'),
url(r'^lang/(?P<language_code>[a-z]{2,5})/?$', views.lang, name='lang'),
url(r'^lang/(?P<language_code>[a-z]{2,5})_(?P<country_code>[A-Z]{2,5})/?$', views.lang, name='lang'),
url(r'^download/?$', views.download, name='download'),
url(r'^download/urls.txt$', views.download_urls_txt, name='download_urls_txt'),
url(r'^download/urls.json$', views.download_urls_json, name='download_urls_json'),
url(r'^download/featured.json$', views.download_featured_json, name='download_featured_json'),
url(r'^download/sites.json$', views.download_sites_json, name='download_sites_json'),
]
@@ -1,8 +1,11 @@
import itertools

from django.shortcuts import render
from django.core.mail import send_mail
from django.utils.html import format_html
from django.http import JsonResponse, HttpResponseNotFound
from django.http import JsonResponse, HttpResponse, HttpResponseNotFound
from django.contrib.auth.decorators import login_required

from .models import Site, Language
from .forms import AddForm

@@ -14,6 +17,7 @@
('/remove/', 'Remove', 'remove'),
('/check/', 'Check sites', 'check'),
('/tos/', 'Terms of Service', 'tos'),
('/downloads/', 'Downloads', 'downloads'),
)


@@ -42,14 +46,14 @@ def index(request):


def lang(request, **filters):
l = Language.objects.filter(**filters)
if not l:
lang = Language.objects.filter(**filters)
if not lang:
return HttpResponseNotFound()
elif len(l) == 1:
lnames = str(l[0])
objects = l[0].site_set
elif len(lang) == 1:
lnames = str(lang[0])
objects = lang[0].site_set
else:
lnames = ', '.join(str(_) for _ in l)
lnames = ', '.join(str(_) for _ in lang)
objects = Site.objects.filter(languages__language_code=filters['language_code'])
return generic_index(request, title="{{0}} Sites in {0}".format(lnames), page='lang', objects=objects)

@@ -184,3 +188,64 @@ def tos(request):
}

return render(request, 'tos.html', context)


def downloads(request):
"""Downloads page."""
all_count = Site.objects.count()
featured_count = Site.objects.filter(featured=True).count()
context = {
'menu': MENU,
'title': 'Downloads',
'page': 'downloads',
'all_count': all_count,
'featured_count': featured_count
}

return render(request, 'downloads.html', context)


def download_urls_txt(request):
"""Download URLs in text format."""
urls = [s.url for s in Site.objects.filter(visible=True).order_by('pk')]
urls.append('') # trailing newline

return HttpResponse('\n'.join(urls), content_type='text/plain')


def download_urls_json(request):
"""Download URLs in JSON format."""
urls = {s.title: s.url for s in Site.objects.filter(visible=True).order_by('pk')}

return JsonResponse(urls)


def download_json_handler(sites):
data = []
for site in sites:
site_info = {
'title': site.title,
'url': site.url,
'author': site.author,
'description': site.description,
'sourcelink': site.sourcelink,
'languages': [lang.code for lang in site.languages.all()],
'featured': site.featured
}
if site.featured:
site_info['featured_order'] = site.featured_order
site_info['featured_reason'] = site.featured_reason
data.append(site_info)

return JsonResponse(data, safe=False)


def download_featured_json(request):
sites = Site.objects.filter(featured=True, visible=True).order_by('featured_order')
return download_json_handler(sites)


@login_required
def download_sites_json(request):
sites = Site.objects.filter(visible=True).order_by('pk')
return download_json_handler(sites)
@@ -0,0 +1,15 @@
{% extends "base.html" %}
{% block content %}
<h2>Downloads</h2>
<p>We offer the ability to download the sites list in multiple formats. The information may only be used for non-commercial purposes.</p>
<ul>
<li><a href="{% url "sites:download_urls_txt" %}">Only URLs, .txt format</a></li>
<li><a href="{% url "sites:download_urls_json" %}">URLs and titles, .json format</a></li>
<li><a href="{% url "sites:download_featured_json" %}">Full data on {{ featured_count }} featured sites, .json format</a></li>
</ul>
<p>If you want information on all {{ all_count }} sites, please contact us (state the reason): <a href="mailto:users@getnikola.com">users@getnikola.com</a>.</p>
{% if user.is_authenticated %}
<a href="{% url "sites:download_sites_json" %}">Download full site data</a> (logged-in users only)
{% endif %}

{% endblock %}

0 comments on commit 29e48f3

Please sign in to comment.