/
decorators.py
73 lines (69 loc) · 1.7 KB
/
decorators.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
import os,cPickle,logging
def write_to_file(path,seperator=' '):
def wrapper(func):
def _func(*args,**karg):
with open(path,'w') as fout:
for sp in func(*args,**karg):
line = seperator.join(sp)
fout.write(line)
fout.write('\n')
return _func
return wrapper
def try_get_path(path):
dirname,filename = os.path.split(path)
if len(filename) <= 220 :
return path
db_path = 'path_mapping.cPickle'
if os.path.exists(db_path) :
with open(db_path) as fin :
db = cPickle.load(fin)
else :
db = {}
if filename not in db :
db[filename] = str(len(db))+'.cPickle'
with open(db_path,'w') as fout :
cPickle.dump(db,fout)
filename = db[filename]
return os.path.join(dirname,filename)
def disk_cached(prefix):
def wrapper(func):
def _func(*args,**karg):
value = None
name = '%s-%s'%(args,karg)
name = name.replace('/','__')
path = '%s%s.cPickle'%(prefix,name)
path = try_get_path(path)
if os.path.exists(path):
logging.info('pickle exists : %s'%(path))
with open(path) as fin:
value = cPickle.load(fin)
logging.info('pickle loaded : %s'%(path))
else :
logging.info('pickle not exists : %s'%(path))
value = func(*args,**karg)
with open(path,'w') as fout:
cPickle.dump(value,fout)
logging.info('pickle written : %s'%(path))
return value
return _func
return wrapper
def memory_cached(func):
cache = {}
list_cache = []
def _func(*args):
value = None
try :
if args in cache:
value = cache[args]
else :
value = func(*args)
cache[args] = value
except :
for k,v in list_cache :
if k==args:
value = v
break
if not value :
value = func(*args)
list_cache.append((args,value))
return value