Provide Link response header for pagination

When traversing the list of snippets page by page provide an additional
response header - Link - that contains the links to the previous and
the next page with the given value of limit (if such pages exist).

This follows the idea from:

    https://developer.github.com/v3/guides/traversing-with-pagination/

and

    https://tools.ietf.org/html/rfc5988

Hopefully, this will make using of pagination easier on the client
side.

tests/resources/test_snippets.py

@@ -320,6 +320,185 @@ async def test_get_snippets_pagination(testapp, snippets, db):
         _compare_snippets(snippet_db, snippet_api)
 
 
+# a helper function of traversing of the list of snippets via API
+async def get_next_page(testapp, limit=3, marker=0):
+    params = []
+    if limit:
+        params.append('limit=%d' % limit)
+    if marker:
+        params.append('marker=%d' % marker)
+
+    resp = await testapp.get(
+        '/snippets' + ('?' if params else '') + '&'.join(params),
+        headers={
+            'Accept': 'application/json',
+            # pass the additional headers, that are set by nginx in the
+            # production deployment, so that we ensure we generate the
+            # correct links for users
+            'Host': 'api.xsnippet.org',
+            'X-Forwarded-Proto': 'https',
+        }
+    )
+    assert resp.status == 200
+    return resp
+
+
+async def test_pagination_links(testapp, db):
+    # put 10 snippets into the db
+    now = datetime.datetime.utcnow().replace(microsecond=0)
+    snippets = [
+        {
+            'id': i + 1,
+            'title': 'snippet #%d' % (i + 1),
+            'content': '(println "Hello, World!")',
+            'syntax': 'clojure',
+            'tags': ['tag_b'],
+            'created_at': (now + datetime.timedelta(seconds=1)),
+            'updated_at': (now + datetime.timedelta(seconds=1)),
+        }
+        for i in range(10)
+    ]
+    await db.snippets.insert(snippets)
+
+    # we should have seen snippets with ids 10, 9 and 8. No link to the prev
+    # page, as we are at the very beginning of the list
+    resp1 = await get_next_page(testapp, limit=3)
+    expected_link1 = (
+        '<https://api.xsnippet.org/snippets>; rel="first", '
+        '<https://api.xsnippet.org/snippets?limit=3&marker=8>; rel="next"'
+    )
+    assert resp1.headers['Link'] == expected_link1
+
+    # we should have seen snippets with ids 7, 6 and 5. Prev page is the
+    # beginning of the list, thus, no marker
+    resp2 = await get_next_page(testapp, limit=3, marker=8)
+    expected_link2 = (
+        '<https://api.xsnippet.org/snippets>; rel="first", '
+        '<https://api.xsnippet.org/snippets?limit=3&marker=5>; rel="next", '
+        '<https://api.xsnippet.org/snippets?limit=3>; rel="prev"'
+    )
+    assert resp2.headers['Link'] == expected_link2
+
+    # we should have seen snippets with ids 4, 3 and 2
+    resp3 = await get_next_page(testapp, limit=3, marker=5)
+    expected_link3 = (
+        '<https://api.xsnippet.org/snippets>; rel="first", '
+        '<https://api.xsnippet.org/snippets?limit=3&marker=2>; rel="next", '
+        '<https://api.xsnippet.org/snippets?limit=3&marker=8>; rel="prev"'
+    )
+    assert resp3.headers['Link'] == expected_link3
+
+    # we should have seen the snippet with id 1. No link to the next page,
+    # as we have reached the end of the list
+    resp4 = await get_next_page(testapp, limit=3, marker=2)
+    expected_link4 = (
+        '<https://api.xsnippet.org/snippets>; rel="first", '
+        '<https://api.xsnippet.org/snippets?limit=3&marker=5>; rel="prev"'
+    )
+    assert resp4.headers['Link'] == expected_link4
+
+
+async def test_pagination_links_one_page_larger_than_whole_list(testapp, db):
+    # put 10 snippets into the db
+    now = datetime.datetime.utcnow().replace(microsecond=0)
+    snippets = [
+        {
+            'id': i + 1,
+            'title': 'snippet #%d' % (i + 1),
+            'content': '(println "Hello, World!")',
+            'syntax': 'clojure',
+            'tags': ['tag_b'],
+            'created_at': (now + datetime.timedelta(seconds=1)),
+            'updated_at': (now + datetime.timedelta(seconds=1)),
+        }
+        for i in range(10)
+    ]
+    await db.snippets.insert(snippets)
+
+    # default limit is 20 and there no prev/next pages - only the first one
+    resp = await get_next_page(testapp, limit=None)
+    expected_link = '<https://api.xsnippet.org/snippets>; rel="first"'
+    assert resp.headers['Link'] == expected_link
+
+
+async def test_pagination_links_port_value_is_preserved_in_url(testapp):
+    # port is omitted
+    resp1 = await testapp.get(
+        '/snippets',
+        headers={
+            'Accept': 'application/json',
+            # pass the additional headers, that are set by nginx in the
+            # production deployment, so that we ensure we generate the
+            # correct links for users
+            'Host': 'api.xsnippet.org',
+            'X-Forwarded-Proto': 'https',
+        }
+    )
+    expected_link1 = '<https://api.xsnippet.org/snippets>; rel="first"'
+    assert resp1.headers['Link'] == expected_link1
+
+    # port is passed explicitly
+    resp2 = await testapp.get(
+        '/snippets',
+        headers={
+            'Accept': 'application/json',
+            # pass the additional headers, that are set by nginx in the
+            # production deployment, so that we ensure we generate the
+            # correct links for users
+            'Host': 'api.xsnippet.org:443',
+            'X-Forwarded-Proto': 'https',
+        }
+    )
+    expected_link2 = '<https://api.xsnippet.org:443/snippets>; rel="first"'
+    assert resp2.headers['Link'] == expected_link2
+
+
+async def test_pagination_links_num_of_items_is_multiple_of_pages(testapp, db):
+    # put 12 snippets into the db
+    now = datetime.datetime.utcnow().replace(microsecond=0)
+    snippets = [
+        {
+            'id': i + 1,
+            'title': 'snippet #%d' % (i + 1),
+            'content': '(println "Hello, World!")',
+            'syntax': 'clojure',
+            'tags': ['tag_b'],
+            'created_at': (now + datetime.timedelta(seconds=1)),
+            'updated_at': (now + datetime.timedelta(seconds=1)),
+        }
+        for i in range(12)
+    ]
+    await db.snippets.insert(snippets)
+
+    # we should have seen snippets with ids 12, 11, 10 and 9. No link to the
+    # prev page, as we are at the very beginning of the list
+    resp1 = await get_next_page(testapp, limit=4)
+    expected_link1 = (
+        '<https://api.xsnippet.org/snippets>; rel="first", '
+        '<https://api.xsnippet.org/snippets?limit=4&marker=9>; rel="next"'
+    )
+    assert resp1.headers['Link'] == expected_link1
+
+    # we should have seen snippets with ids 8, 7, 6 and 5. Link to the prev
+    # page is a link to the beginning of the list
+    resp2 = await get_next_page(testapp, limit=4, marker=9)
+    expected_link2 = (
+        '<https://api.xsnippet.org/snippets>; rel="first", '
+        '<https://api.xsnippet.org/snippets?limit=4&marker=5>; rel="next", '
+        '<https://api.xsnippet.org/snippets?limit=4>; rel="prev"'
+    )
+    assert resp2.headers['Link'] == expected_link2
+
+    # we should have seen snippets with ids 4, 3, 2 and 1. Link to the next
+    # page is not rendered, as we reached the end of the list
+    resp3 = await get_next_page(testapp, limit=4, marker=5)
+    expected_link3 = (
+        '<https://api.xsnippet.org/snippets>; rel="first", '
+        '<https://api.xsnippet.org/snippets?limit=4&marker=9>; rel="prev"'
+    )
+    assert resp3.headers['Link'] == expected_link3
+
+
 async def test_get_snippets_pagination_not_found(testapp):
     resp = await testapp.get(
         '/snippets?limit=10&marker=1234567890',

xsnippet/api/resource.py

@@ -58,7 +58,7 @@ def __init__(self, *args, **kwargs):
         #: an application database alias to make code a bit readable
         self.db = self.request.app['db']
 
-    def make_response(self, data, status=200):
+    def make_response(self, data, headers=None, status=200):
         """Return an HTTP response object.
 
         The method serializes given data according to 'Accept' HTTP header,
@@ -67,6 +67,9 @@ def make_response(self, data, status=200):
         :param data: a data to be responded to client
         :type data: a serializable python object
 
+        :param headers: response headers to be returned to client
+        :type headers: dict
+
         :param status: an HTTP status code
         :type status: int
 
@@ -99,6 +102,7 @@ def make_response(self, data, status=200):
                 if fnmatch.fnmatch(supported_accept, accept):
                     return web.Response(
                         status=status,
+                        headers=headers,
                         content_type=supported_accept,
                         text=encode(data),
                     )

xsnippet/api/resources/snippets.py

@@ -130,6 +130,58 @@ async def service_fn(snippet):
         return await _write(self, service_fn, status=200)
 
 
+def _build_url_from_marker(request, marker, limit=20):
+    # take into account, that we might be running behind a reverse proxy
+    host = request.headers.get('Host', request.url.host)
+    if ':' in host:
+        host, _ = host.split(':')
+    proto = request.headers.get('X-Forwarded-Proto', request.scheme)
+
+    # drop the previous values of limit and marker
+    new_query = request.url.query.copy()
+    new_query.pop('limit', None)
+    new_query.pop('marker', None)
+
+    # and replace them with new ones (if necessary for the given page)
+    if limit:
+        new_query['limit'] = limit
+    if marker:
+        new_query['marker'] = marker
+
+    return request.url.with_scheme(proto) \
+                      .with_host(host) \
+                      .with_query(new_query)
+
+
+def _build_link_header(request, current_page, previous_page, limit=20):
+    links = []
+
+    # always render a link to the first page
+    url = _build_url_from_marker(request, marker=None, limit=None)
+    links.append('<%s>; rel="first"' % url)
+
+    # only render a link, if there more items after the current page
+    if len(current_page) > limit:
+        marker = current_page[:limit][-1]['id']
+        url = _build_url_from_marker(request, marker, limit)
+
+        links.append('<%s>; rel="next"' % url)
+
+    if len(previous_page) >= limit:
+        # account for the edge case when there is exactly one full page left -
+        # in this case the request for the previous page should be w/o any
+        # marker value passed
+        if len(previous_page) == limit:
+            marker = None
+        else:
+            marker = previous_page[-1]['id']
+        url = _build_url_from_marker(request, marker, limit)
+
+        links.append('<%s>; rel="prev"' % url)
+
+    return ', '.join(links)
+
+
 class Snippets(resource.Resource):
 
     async def get(self):
@@ -170,20 +222,50 @@ async def get(self):
             error = '%s.' % cerberus_errors_to_str(v.errors)
             return self.make_response({'message': error}, status=400)
 
+        # It's safe to have type cast here since those query parameters
+        # are guaranteed to be integer, thanks to validation above.
+        limit = int(self.request.GET.get('limit', 20))
+        marker = int(self.request.GET.get('marker', 0))
+        title = self.request.GET.get('title')
+        tag = self.request.GET.get('tag')
+        syntax = self.request.GET.get('syntax')
+
         try:
-            snippets = await services.Snippet(self.db).get(
-                title=self.request.GET.get('title'),
-                tag=self.request.GET.get('tag'),
-                syntax=self.request.GET.get('syntax'),
-                # It's safe to have type cast here since those query parameters
-                # are guaranteed to be integer, thanks to validation above.
-                limit=int(self.request.GET.get('limit', 0)),
-                marker=int(self.request.GET.get('marker', 0)),
+            # actual snippets to be returned
+            current_page = await services.Snippet(self.db).get(
+                title=title, tag=tag, syntax=syntax,
+                # read one more to know if there is next page
+                limit=limit + 1,
+                marker=marker,
             )
+
+            # only needed to render a link to the previous page
+            if marker:
+                previous_page = await services.Snippet(self.db).get(
+                    title=title, tag=tag, syntax=syntax,
+                    # read one more to know if there is prev page
+                    limit=limit + 1,
+                    marker=marker,
+                    direction='backward'
+                )
+            else:
+                # we are at the very beginning of the list - no prev page
+                previous_page = []
         except exceptions.SnippetNotFound as exc:
             return self.make_response({'message': str(exc)}, status=404)
 
-        return self.make_response(snippets, status=200)
+        return self.make_response(
+            # return no more than $limit snippets (if we read $limit + 1)
+            current_page[:limit],
+            headers={
+                'Link': _build_link_header(
+                    self.request,
+                    current_page, previous_page,
+                    limit=limit,
+                )
+            },
+            status=200
+        )
 
     async def post(self):
         return await _write(self, services.Snippet(self.db).create, status=201)