diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..9a02278e --- /dev/null +++ b/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2012, Roman Haritonov +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/docs/api.rst b/docs/api.rst index 7cdad8dd..e52020a2 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1,7 +1,7 @@ API === -This part of the documentation covers all the interfaces of :mod:`requsts_cache` +This part of the documentation covers all the interfaces of `requests-cache` Public api @@ -11,8 +11,17 @@ Public api :members: -Extensible objects ------------------- +Cache backends +-------------- .. automodule:: requests_cache.backends.base :members: + +.. automodule:: requests_cache.backends.sqlite + :members: + +Internal modules which can be used outside +------------------------------------------ + +.. automodule:: requests_cache.backends.dbdict + :members: diff --git a/requests_cache/backends/base.py b/requests_cache/backends/base.py index 8a991890..9e7af3c9 100644 --- a/requests_cache/backends/base.py +++ b/requests_cache/backends/base.py @@ -4,7 +4,8 @@ requests_cache.backends.base ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Contains base extensible in-memory cache backend and common functions + Contains in-memory cache backend which can be extended to support persistence + and some common functions """ from datetime import datetime import requests @@ -31,6 +32,8 @@ def save_response(self, url, response): .. note:: urls from history saved automatically :param response: response to save + + .. note:: Response is reduced before saving (with :func:`reduce_response`) to make it picklable """ self.responses[url] = reduce_response(response), datetime.now() self.url_map[url] = response.url @@ -44,6 +47,8 @@ def get_response_and_time(self, url, default=(None, None)): :param url: url of resource :param default: return this if `url` not found in cache :returns: tuple (response, datetime) + + .. note:: Response is restored after unpickling with :func:`restore_response` """ try: response, timestamp = self.responses[self.url_map[url]] @@ -80,6 +85,8 @@ class _Store(object): 'headers', 'encoding') def reduce_response(response): + """ Reduce response object to make it compatible with pickle + """ result = _Store() # prefetch response.content @@ -91,6 +98,8 @@ def reduce_response(response): return result def restore_response(response): + """ Restore response object after unpickling + """ result = requests.Response() for field in _fields_to_copy: setattr(result, field, getattr(response, field)) diff --git a/requests_cache/backends/dbdict.py b/requests_cache/backends/dbdict.py index 0f523bf3..7cb6ab6e 100644 --- a/requests_cache/backends/dbdict.py +++ b/requests_cache/backends/dbdict.py @@ -1,13 +1,23 @@ #!/usr/bin/python # -*- coding: utf-8 -*- +""" + requests_cache.backends.dbdict + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Dictionary-like objects for saving large data sets to `sqlite` database +""" import UserDict import pickle import sqlite3 as sqlite class DbDict(object, UserDict.DictMixin): - """ DbDict - a dictionary-like object for large datasets with sqlite backend""" + """ DbDict - a dictionary-like object for saving large datasets to `sqlite` database + """ def __init__(self, dict_name): + """ + :param dict_name: filename for database + """ self.db_filename = "%s.sqlite" % dict_name self.con = sqlite.connect(self.db_filename) self.con.execute("create table if not exists data (key PRIMARY KEY, value)") @@ -41,13 +51,12 @@ def clear(self): self.con.execute("create table data (key PRIMARY KEY, value)") self.con.commit() - def __str__(self): return str(dict(self.iteritems())) class DbPickleDict(DbDict): - """ Pickles data before store + """ Same as :class:`DbDict`, but pickles values before saving """ def __setitem__(self, key, item): super(DbPickleDict, self).__setitem__(key, sqlite.Binary(pickle.dumps(item))) diff --git a/requests_cache/backends/sqlite.py b/requests_cache/backends/sqlite.py index 95ee4c9a..f6b21911 100644 --- a/requests_cache/backends/sqlite.py +++ b/requests_cache/backends/sqlite.py @@ -1,13 +1,25 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import requests +""" + requests_cache.backends.sqlite + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ``sqlite3`` cache backend +""" from requests_cache.backends.base import MemoryCache from requests_cache.backends.dbdict import DbPickleDict class DbCache(MemoryCache): - """ sqlite cache backend + """ sqlite cache backend. + + It stores cache data to two files (for ``location = 'cache'``): + + - ``cache_urls.sqlite`` + - ``cache_responses.sqlite`` + + Reading is fast, saving is bit slower. It can store big amount of data + with low memory usage. """ def __init__(self, location='cache', *args, **kwargs): super(DbCache, self).__init__(*args, **kwargs) diff --git a/requests_cache/core.py b/requests_cache/core.py index f5530f3f..edaaf7f7 100644 --- a/requests_cache/core.py +++ b/requests_cache/core.py @@ -19,23 +19,24 @@ _cache = None -def configure(cache_filename_prefix='cache', backend='sqlite', expire_after=60, +def configure(cache_name_prefix='cache', backend='sqlite', expire_after=60, allowable_codes=(200,), monkey_patch=True): - """ Configure cache and patch ``requests`` library, to transparently use it + """ + Configure cache storage and patch ``requests`` library to transparently cache responses - :param cache_filename_prefix: cache files will start with this prefix, + :param cache_name_prefix: cache files will start with this prefix, e.g ``cache_urls.sqlite``, ``cache_responses.sqlite`` :param backend: cache backend e.g ``'sqlite'``, ``'memory'`` - :param expire_after: number of minutes after cache will be expired + :param expire_after: number of minutes after cache will be expired (default 60) :type expire_after: int or float :param allowable_codes: limit caching only for response with this codes :type allowable_codes: tuple - :param monkey_patch: patch ``requests.Request.send`` if True, otherwise + :param monkey_patch: patch ``requests.Request.send`` if `True` (default), otherwise cache will not work until calling :func:`redo_patch` """ try: global _cache - _cache = backends.registry[backend](cache_filename_prefix) + _cache = backends.registry[backend](cache_name_prefix) except KeyError: raise ValueError('Unsupported backend "%s" try one of: %s' % (backend, ', '.join(backends.registry.keys())))