Skip to content

Commit

Permalink
fix bang gap sample
Browse files Browse the repository at this point in the history
  • Loading branch information
@TsumiNa
TsumiNa committed Jan 23, 2018
1 parent 9909182 commit 218da75
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 115 deletions.
94 changes: 28 additions & 66 deletions samples/experiment_band_gap_prediction.ipynb
View file

Large diffs are not rendered by default.

42 changes: 24 additions & 18 deletions tests/test_datatools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ def setup():
# prepare path
test = dict(
test_file=Path().cwd() / 'fetch_test.txt',
fetch_file='https://raw.githubusercontent.com/yoshida-lab/XenonPy/master/travis/fetch_test.txt',
fetch_file=
'https://raw.githubusercontent.com/yoshida-lab/XenonPy/master/travis/fetch_test.txt',
test_dir='test_dir',
user_dataset='test_user_data'
)
user_dataset='test_user_data')

# ignore numpy warning
import warnings
Expand Down Expand Up @@ -50,18 +50,22 @@ def test_loader(setup):


def test_loader_url(setup):
load = Loader('https://raw.githubusercontent.com/yoshida-lab/XenonPy/master/')
load = Loader(
'https://raw.githubusercontent.com/yoshida-lab/XenonPy/master/')
file = load('travis/fetch_test.txt')
with open(str(file)) as f:
assert f.readline() == 'Test xenonpy.utils.Loader._fetch_data'


def test_loader_property(setup):
load = Loader()
assert 118 == load.elements.shape[0], 'should have 118 elements'
assert 74 == load.elements.shape[1], 'should have 74 features'

assert 94 == load.elements_completed.shape[0], 'should have 94 completed elements'
assert 58 == load.elements_completed.shape[1], 'should have 58 completed features'
assert 94 == load.elements_completed.shape[
0], 'should have 94 completed elements'
assert 58 == load.elements_completed.shape[
1], 'should have 58 completed features'


def test_loader_return_saver(setup):
Expand All @@ -87,31 +91,33 @@ def test__fetch_data(setup):

# =====================================================================================================


def test_saver1(setup):
saver = Saver(setup['user_dataset'])
assert len(saver._files) == 0, 'no files'
ret = 'Dataset: {} with:'.format(setup['user_dataset'])
assert str(saver) == ret, 'no files'


def test_saver2(setup):
saver = Saver(setup['user_dataset'])
saver(list('abcd'), list('efgh'))
assert len(saver._files) == 2, 'should got 2 files'
assert len(saver._files['temp_data']) == 2, 'should got 2 files'


def test_saver3(setup):
saver = Saver(setup['user_dataset'])
saver(key1=list('asdf'), key2=list('qwer'))
assert len(saver._files) == 2, 'should got 2 files'
assert len(saver._name_files['key1']) == 1, 'should got 1 files'
assert len(saver._name_files['key2']) == 1, 'should got 1 files'
assert len(saver._files['temp_data']) == 2, 'should got 2 files'
assert len(saver._files['key1']) == 1, 'should got 1 files'
assert len(saver._files['key2']) == 1, 'should got 1 files'


def test_saver4(setup):
saver = Saver(setup['user_dataset'])
saver(list('asdf'), key1=list('qwer'))
assert len(saver._files) == 3, 'should got 3 files'
assert len(saver._name_files['key1']) == 2, 'should got 1 files'
assert len(saver._name_files['key2']) == 1, 'should got 1 files'
assert len(saver._files['temp_data']) == 3, 'should got 3 files'
assert len(saver._files['key1']) == 2, 'should got 1 files'
assert len(saver._files['key2']) == 1, 'should got 1 files'


def test_saver_last1(setup):
Expand Down Expand Up @@ -145,22 +151,22 @@ def test_saver_getitem2(setup):
def test_saver_delete1(setup):
saver = Saver(setup['user_dataset'])
saver.rm(0)
assert len(saver._files) == 2, 'should got 1 files'
assert len(saver._files['temp_data']) == 2, 'should got 1 files'


def test_saver_delete2(setup):
saver = Saver(setup['user_dataset'])
saver(key1=list('qwer'))
assert len(saver._name_files['key1']) == 3, 'should got 3 files'
assert len(saver._files['key1']) == 3, 'should got 3 files'
saver.rm(slice(0, 2), 'key1')
assert len(saver._name_files['key1']) == 1, 'should got 1 files'
assert len(saver._files['key1']) == 1, 'should got 1 files'


def test_saver_clean1(setup):
saver = Saver(setup['user_dataset'])
saver_dir = Path.home() / '.xenonpy' / 'userdata' / setup['user_dataset']
saver.clean('key1')
assert 'key1' not in saver._name_files, 'no saver dir'
assert 'key1' not in saver._files, 'no saver dir'


def test_saver_clean2(setup):
Expand Down
57 changes: 26 additions & 31 deletions xenonpy/utils/datatools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -295,26 +295,23 @@ def __init__(self, dataset=None):
if not self._path.exists():
self._path.mkdir(parents=True)
self._files = None
self._name_files = None
self._make_file_index()

def _make_file_index(self):
self._files = list()
self._name_files = defaultdict(list)
self._files = defaultdict(list)
files = [f for f in self._path.iterdir() if f.match('*.pkl.*')]

for f in files:
# named data
if len(f.suffixes) == 3:
dname = f.suffixes[0].lstrip('.')
self._name_files[dname].append(f)
d_name = f.suffixes[0].lstrip('.')
self._files[d_name].append(f)
continue

# unnamed data
self._files.append(f)
self._files['temp_data'].append(f)

self._sort_files(self._files)
for v in self._name_files.values():
for v in self._files.values():
self._sort_files(v)

@classmethod
Expand Down Expand Up @@ -345,53 +342,53 @@ def _save_data(self, data, filename=None):

return file

def last(self, data_name: str = None):
def last(self, d_name: str = None):
"""
Return last saved data.
Args
----
data_name: str
Data's name. Omit for access unnamed data
d_name: str
Data's name. Omit for access temp data
Return
-------
ret:any python object
Data stored in `*.pkl` file.
"""
if data_name is None:
return self._load_data(self._files[-1])
return self._load_data(self._name_files[data_name][-1])
if d_name is None:
return self._load_data(self._files['temp_data'][-1])
return self._load_data(self._files[d_name][-1])

def rm(self, index, dname: str = None):
def rm(self, index, d_name: str = None):
"""
Delete file(s) with given index.
Parameters
----------
index: int or slice
Index of data. Data sorted by datetime.
dname: str
d_name: str
Data's name. Omit for access unnamed data.
"""

if not dname:
files = self._files[index]
if not d_name:
files = self._files['temp_data'][index]
if not isinstance(files, list):
remove(str(files))
else:
for f in files:
remove(str(f))
del self._files[index]
del self._files['temp_data'][index]
return

files = self._name_files[dname][index]
files = self._files[d_name][index]
if not isinstance(files, list):
remove(str(files))
else:
for f in files:
remove(str(f))
del self._name_files[dname][index]
del self._files[d_name][index]

def clean(self, data_name: str = None):
"""
Expand All @@ -405,20 +402,18 @@ def clean(self, data_name: str = None):
if data_name is None:
rmtree(str(self._path))
self._files = list()
self._name_files = defaultdict(list)
self._files = defaultdict(list)
return

for f in self._name_files[data_name]:
for f in self._files[data_name]:
remove(str(f))
del self._name_files[data_name]
del self._files[data_name]

def __repr__(self):
cont_ls = []

cont_ls.append('Dataset: {} with:'.format(self.dataset))
cont_ls.append('tempfiles: {}'.format(len(self._files)))

for k, v in self._name_files.items():
for k, v in self._files.items():
cont_ls.append('{}: {}'.format(k, len(v)))

return '\n'.join(cont_ls)
Expand All @@ -440,20 +435,20 @@ def _load_file(files, item):
if not isinstance(key, str) or \
(not isinstance(index, int) and not isinstance(index, slice)):
raise ValueError('except 2 parameters. [str, int or slice]')
return _load_file(self._name_files[key], index)
return _load_file(self._files[key], index)

if isinstance(item, str):
return self.__getitem__((item, slice(None, None, None)))

return _load_file(self._files, item)
return _load_file(self._files['temp_data'], item)

def __call__(self, *data, **named_data):
for d in data:
f = self._save_data(d)
self._files.append(f)
self._files['temp_data'].append(f)

for k, v in named_data.items():
f = self._save_data(v, k)
self._name_files[k].append(f)
self._files[k].append(f)

return

0 comments on commit 218da75

Please sign in to comment.