diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..a7bac06 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,21 @@ +language: python +python: + - "2.6" + - "2.7" + - "3.2" + - "3.3" + - "3.4" + - "3.5" + - "3.5-dev" # 3.5 development branch + - "3.6" + - "3.6-dev" # 3.6 development branch + - "3.7-dev" # 3.7 development branch + - "nightly" # currently points to 3.7-dev + # PyPy versions + - "pypy" # PyPy2 2.5.0 + - "pypy3" # Pypy3 2.4.0 + - "pypy-5.3.1" +install: + - pip install . + - pip install -r requirements.txt +script: pytest diff --git a/pyspeedup/algorithms/Primes.py b/pyspeedup/algorithms/Primes.py index df9308a..f72ae3f 100644 --- a/pyspeedup/algorithms/Primes.py +++ b/pyspeedup/algorithms/Primes.py @@ -5,7 +5,10 @@ and uses concurrent branching to split primality tests and prime factorizations and speed up the whole process. """ -import cPickle +try: + import cPickle as pickle +except: + import pickle from os.path import join from threading import Thread from time import sleep @@ -20,8 +23,8 @@ global q global thread global file_location - D = DiskDict() - F = OrderedDiskDict() + D = DiskDict("seive") + F = OrderedDiskDict("factors") c = 3 p = 3 q = 9 @@ -42,14 +45,12 @@ def load_primes(location): global file_location stop_seive() del D - D = DiskDict() #New seive object. - D.link_to_disk("seive",file_location=location, size_limit = 65536, max_pages = 32) #Load or create persistance. + D = DiskDict("seive", file_location=location, size_limit = 65536, max_pages = 32) #New seive object. del F - F = OrderedDiskDict() #A new factor list object. - F.link_to_disk("factors",file_location=location, size_limit = 65536, max_pages = 32) #Load or create persistance. + F = OrderedDiskDict("factors",file_location=location, size_limit = 65536, max_pages = 32) #A new factor list object. try: with open(D._file_base+"current",'rb') as f: - c,p = cPickle.load(f) + c,p = pickle.load(f) except: c,p = 3,3 file_location = location @@ -107,7 +108,7 @@ def _prime_seive(): c += 2 #We skip all even numbers since 2 is hard coded. sleep(0) with open(D._file_base+"current",'wb') as f: - cPickle.dump((c,p), f) + pickle.dump((c,p), f) def nextPrime(p): """ @@ -212,7 +213,7 @@ def get_factorization(q): from os.path import expanduser load_primes("D:/.pyspeedup") start_seive() - while len(F)<1000000: + while len(F)<1000000: sleep(1) stop_seive() print(c) diff --git a/pyspeedup/algorithms/__init__.py b/pyspeedup/algorithms/__init__.py index 427f721..e028d6c 100644 --- a/pyspeedup/algorithms/__init__.py +++ b/pyspeedup/algorithms/__init__.py @@ -21,4 +21,4 @@ class which I intend to build. from pyspeedup.algorithms._indexCalculus import rowReduce from pyspeedup.algorithms._squares import isSquare from pyspeedup.algorithms._factor import factor -import Primes as Primes \ No newline at end of file +from pyspeedup.algorithms import Primes diff --git a/pyspeedup/memory/_diskdict.py b/pyspeedup/memory/_diskdict.py index 803a241..cd6c2ed 100644 --- a/pyspeedup/memory/_diskdict.py +++ b/pyspeedup/memory/_diskdict.py @@ -1,5 +1,8 @@ from collections import MutableMapping -import pickle +try: + import cPickle as pickle +except: + import pickle from os.path import expanduser,join from os import remove,makedirs from glob import glob @@ -235,7 +238,7 @@ def _load_page_from_disk(self,number): def __str__(self): return "Dictionary with values stored to "+self._file_base def __repr__(self): - return "DiskDict().link_to_disk('',"+str(self.size_limit)+','+str(self.max_pages)+','+self._file_base+')' + return "DiskDict('',"+str(self.size_limit)+','+str(self.max_pages)+','+self._file_base+')' def __contains__(self, item): i,k = self._finditem(item) return k in self.pages[i] diff --git a/pyspeedup/memory/_disklist.py b/pyspeedup/memory/_disklist.py index 82a3c2a..9973c3d 100644 --- a/pyspeedup/memory/_disklist.py +++ b/pyspeedup/memory/_disklist.py @@ -1,5 +1,8 @@ from collections import MutableSequence -import pickle +try: + import cPickle +except: + import pickle from os.path import expanduser,join from os import remove, makedirs from glob import glob diff --git a/pyspeedup/memory/_ordereddiskdict.py b/pyspeedup/memory/_ordereddiskdict.py index 36b9ec5..fe49aed 100644 --- a/pyspeedup/memory/_ordereddiskdict.py +++ b/pyspeedup/memory/_ordereddiskdict.py @@ -1,5 +1,8 @@ from collections import MutableMapping -import cPickle +try: + import cPickle as pickle +except: + import pickle from os.path import expanduser,join from os import remove,makedirs from glob import glob @@ -23,31 +26,28 @@ class OrderedDiskDict(MutableMapping): This is accomplished through a rudimentary (for now) hashing scheme to page the dictionary into parts. """ - def __init__(self, *args, **kwargs): - self.pages=_page() - self.pages[0]=_page(*args, **kwargs) - self._length = len(self.pages[0]) - self._total={0} - self._queue=[0] - self._file_base = None - def link_to_disk(self, file_basename, size_limit = 1024, max_pages = 16, file_location = join(expanduser("~"),"PySpeedup")): - if len(self)>0: - raise Exception("Linking to disk should happen before any data is written.") - if self._file_base: - raise Exception("Can't link to two file names or locations at the same time.") - try: - os.makedirs(file_location) - except: - pass - self._file_base = join(file_location,file_basename) - self.size_limit = size_limit + def __init__(self, file_basename, size_limit = 1024, max_pages = 16, file_location = join(expanduser("~"),"PySpeedup")): + if max_pages < 1: + raise ValueError("There must be allowed at least one page in RAM.") self.max_pages = max_pages + if size_limit < 1: + raise ValueError("There must be allowed at least one item per page.") + self.size_limit = size_limit + if file_location: + try: + makedirs(file_location) + except OSError as e: + if e.errno != 17: + raise + pass + self._file_base = join(file_location,file_basename) + self.pages = _page() + self._length = 0 + self._total = set() + self._queue = [] try: with open(self._file_base+'Len', 'rb') as f: - self._length = cPickle.load(f) - self._total.remove(0) - self._queue=[] - del self.pages[0] + self.pages.currentDepth,self._length = pickle.load(f) for f in glob(self._file_base+'*'): try: self._total.add(int(f[len(self._file_base):])) @@ -56,7 +56,6 @@ def link_to_disk(self, file_basename, size_limit = 1024, max_pages = 16, file_lo except: pass atexit.register(_exitgracefully,self) - return self def _guarantee_page(self,k): """ Pulls up the page in question. @@ -143,14 +142,14 @@ def __del__(self): for key in self.pages.keys(): self._save_page_to_disk(key) def _save_page_to_disk(self,number): - import cPickle + import pickle with open(self._file_base+'Len', 'wb') as f: - cPickle.dump(self._length,f) + pickle.dump(self._length,f) if self._file_base: if number in self.pages: if len(self.pages[number])>0: with open(self._file_base+str(number),'wb') as f: - cPickle.dump(self.pages[number],f) + pickle.dump(self.pages[number],f) else: self._total.remove(number) del self.pages[number] @@ -161,7 +160,7 @@ def _save_page_to_disk(self,number): def _load_page_from_disk(self,number): if self._file_base: with open(self._file_base+str(number),'rb') as f: - self.pages[number] = cPickle.load(f) + self.pages[number] = pickle.load(f) self._queue.append(number) remove(self._file_base+str(number)) def __str__(self): @@ -178,12 +177,11 @@ def __exit__(self, exception_type, exception_val, trace): if __name__ == '__main__': - d = DiskDict() - d.link_to_disk('testDiskDict',2,2) + d = DiskDict('testDiskDict',2,2) for i in range(16): d[i/10.]=i print(d.pages) d.max_pages=16 for i in range(16): d[i/10.]=i - print(d.pages) \ No newline at end of file + print(d.pages) diff --git a/setup.py b/setup.py index b2e7ff6..30a8c60 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ def read(fname): setup( name = "PySpeedup", - version = "0.1.2.2", + version = "0.1.3.0", author = "Chris Dusold", author_email = "PySpeedup@ChrisDusold.com", description = ("A multiprocess framework for efficient calculations."), diff --git a/source/index.rst b/source/index.rst index 3e0bab3..147d82e 100644 --- a/source/index.rst +++ b/source/index.rst @@ -80,6 +80,11 @@ Disk Based Dictionary .. autoclass:: pyspeedup.memory.DiskDict +Ordered Access Disk Based Dictionary +--------------------- + +.. autoclass:: pyspeedup.memory.OrderedDiskDict + Disk Based List --------------- diff --git a/tests/cacheTests.py b/tests/test_cache.py similarity index 100% rename from tests/cacheTests.py rename to tests/test_cache.py