-
Notifications
You must be signed in to change notification settings - Fork 0
/
JsonPydexer.py
78 lines (67 loc) · 2.77 KB
/
JsonPydexer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
__author__ = "Christian Bailey (me@christianbailey.me)"
__version__ = "0.2.0"
import os
import json
import pickle
import sys
import stat
from functools import reduce
from operator import getitem
class JsonPydexer:
"""Base class for doing things"""
def __init__(self, path):
"""Initialize class and check path
Args:
self: self
path (str): path of directory containing files to index
Returns:
JsonPydexer: instance
"""
self.rootPath = path
# check that path is actually a directory
if (stat.S_ISDIR(os.stat(self.rootPath).st_mode) == 0):
print("Error: {} is not reported as a directory.".format(self.rootPath), file=sys.stderr)
raise ValueError("A non-directory file was passed")
elif not (os.access(self.rootPath, os.W_OK)):
raise ValueError("Specified directory is not writable")
def index(self, key, r=False, filename=None):
"""Index .json files in the root path. Pickles the index as a dict, with
whichever key you specify as the key and relative filenames as values to
save on storage space.
Args:
key (list): list of keys. each element of the list is one level of a
nested key
r (bool, optional): recurse into lower directories or not. not yet
implemented
filename (string, optional): Filename to save the index as. defaulti
is str(key) + ".pickle"
Returns: None
"""
if type(key) is str:
key = [key]
if filename is None:
filename = ''.join(key + [".pickle"])
#TODO check if filename already exists,
index = dict()
if r:
for root, subFolders, files in os.walk(self.rootPath):
for file in files:
name, extension = os.path.splitext(file)
if extension == ".json":
with open(root + "/" + file) as f:
j = json.load(f)
file_key = reduce(getitem, key, j)
index[file_key] = os.path.join(
os.path.relpath(root, start=self.rootPath), file
)
else:
for root, subFolders, files in os.walk(self.rootPath):
for file in files:
name, extension = os.path.splitext(file)
if extension == ".json":
with open(root + "/" + file) as f:
j = json.load(f)
file_key = reduce(getitem, key, j)
index[file_key] = file
with open(filename, mode="wb") as f:
pickle.dump(index, f)