/
path.py
492 lines (356 loc) · 14.8 KB
/
path.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
# coding: utf-8
# Copyright (c) Max-Planck-Institut für Eisenforschung GmbH - Computational Materials Design (CM) Department
# Distributed under the terms of "New BSD License", see the LICENSE file.
import os
import posixpath
from pyiron.base.generic.hdfio import ProjectHDFio
from pyiron.base.job.core import JobCore
from pyiron.base.project.generic import Project
"""
The JobPath class enables quick access to the HDF5 data file without loading the full object
"""
__author__ = "Joerg Neugebauer, Jan Janssen"
__copyright__ = (
"Copyright 2020, Max-Planck-Institut für Eisenforschung GmbH - "
"Computational Materials Design (CM) Department"
)
__version__ = "1.0"
__maintainer__ = "Jan Janssen"
__email__ = "janssen@mpie.de"
__status__ = "production"
__date__ = "Sep 1, 2017"
class JobPathBase(JobCore):
"""
The JobPath class is derived from the JobCore and is used as a lean version of the GenericJob class. Instead of
loading the full pyiron object the JobPath class only provides access to the HDF5 file, which should be enough
for most analysis.
Args:
db (DatabaseAccess): database object
job_id (int): Job ID - optional, but either a job ID or a database entry db_entry has to be provided.
db_entry (dict): database entry {"job":, "subjob":, "projectpath":, "project":, "hamilton":, "hamversion":,
"status":} and optional entries are {"id":, "masterid":, "parentid":}
user (str): current unix/linux/windows user who is running pyiron
Attributes:
.. attribute:: job_name
name of the job, which has to be unique within the project
.. attribute:: status
execution status of the job, can be one of the following [initialized, appended, created, submitted, running,
aborted, collect, suspended, refresh, busy, finished]
.. attribute:: job_id
unique id to identify the job in the pyiron database
.. attribute:: parent_id
job id of the predecessor job - the job which was executed before the current one in the current job series
.. attribute:: master_id
job id of the master job - a meta job which groups a series of jobs, which are executed either in parallel or in
serial.
.. attribute:: child_ids
list of child job ids - only meta jobs have child jobs - jobs which list the meta job as their master
.. attribute:: project
Project instance the jobs is located in
.. attribute:: project_hdf5
ProjectHDFio instance which points to the HDF5 file the job is stored in
.. attribute:: job_info_str
short string to describe the job by it is job_name and job ID - mainly used for logging
.. attribute:: working_directory
working directory of the job is executed in - outside the HDF5 file
.. attribute:: path
path to the job as a combination of absolute file system path and path within the HDF5 file.
.. attribute:: is_root
boolean if the HDF5 object is located at the root level of the HDF5 file
.. attribute:: is_open
boolean if the HDF5 file is currently opened - if an active file handler exists
.. attribute:: is_empty
boolean if the HDF5 file is empty
.. attribute:: base_name
name of the HDF5 file but without any file extension
.. attribute:: file_path
directory where the HDF5 file is located
.. attribute:: h5_path
path inside the HDF5 file - also stored as absolute path
"""
def __init__(self, job_path):
job_path_lst = job_path.replace("\\", "/").split(".h5")
if len(job_path_lst) != 2:
raise ValueError
sub_job = job_path_lst[1]
h5_path = None
if sub_job is not None:
if len(sub_job.strip()) > 0:
h5_path = "/".join(sub_job.split("/")[:-1])
hdf_project = ProjectHDFio(
project=Project(os.path.dirname(job_path_lst[0])),
file_name=job_path_lst[0].split("/")[-1] + ".h5",
h5_path=h5_path,
mode="r",
)
super(JobPathBase, self).__init__(
project=hdf_project, job_name=job_path_lst[1].split("/")[-1]
)
@property
def is_root(self):
"""
Check if the current h5_path is pointing to the HDF5 root group.
Returns:
bool: [True/False]
"""
return self.project_hdf5.is_root
@property
def is_empty(self):
"""
Check if the HDF5 file is empty
Returns:
bool: [True/False]
"""
return self.project_hdf5.is_empty
@property
def base_name(self):
"""
Name of the HDF5 file - but without the file extension .h5
Returns:
str: file name without the file extension
"""
return self.project_hdf5.base_name
@property
def file_path(self):
"""
Path where the HDF5 file is located - posixpath.dirname()
Returns:
str: HDF5 file location
"""
return self.project_hdf5.file_path
@property
def h5_path(self):
"""
Get the path in the HDF5 file starting from the root group - meaning this path starts with '/'
Returns:
str: HDF5 path
"""
return self.project_hdf5.h5_path
@h5_path.setter
def h5_path(self, path):
"""
Set the path in the HDF5 file starting from the root group
Args:
path (str): HDF5 path
"""
self.project_hdf5.h5_path = path
def create_group(self, name):
"""
Create an HDF5 group - similar to a folder in the filesystem - the HDF5 groups allow the users to structure their
data.
Args:
name (str): name of the HDF5 group
Returns:
FileHDFio: FileHDFio object pointing to the new group
"""
return self.project_hdf5.create_group(name)
def open(self, h5_rel_path):
"""
Create an HDF5 group and enter this specific group. If the group exists in the HDF5 path only the h5_path is
set correspondingly otherwise the group is created first.
Args:
h5_rel_path (str): relative path from the current HDF5 path - h5_path - to the new group
Returns:
FileHDFio: FileHDFio object pointing to the new group
"""
return self.project_hdf5.open(h5_rel_path)
def close(self):
"""
Close the current HDF5 path and return to the path before the last open
"""
self.project_hdf5.close()
def remove_file(self):
"""
Remove the HDF5 file with all the related content
"""
self.project_hdf5.remove_file()
def put(self, key, value):
"""
Store data inside the HDF5 file
Args:
key (str): key to store the data
value (pandas.DataFrame, pandas.Series, dict, list, float, int): basically any kind of data is supported
"""
self.project_hdf5.__setitem__(key, value)
def listdirs(self):
"""
equivalent to os.listdirs (consider groups as equivalent to dirs)
Returns:
(list): list of groups in pytables for the path self.h5_path
"""
return self.project_hdf5.list_groups()
def list_dirs(self):
"""
equivalent to os.listdirs (consider groups as equivalent to dirs)
Returns:
(list): list of groups in pytables for the path self.h5_path
"""
return self.project_hdf5.list_groups()
def keys(self):
"""
List all groups and nodes of the HDF5 file - where groups are equivalent to directories and nodes to files.
Returns:
list: all groups and nodes
"""
return self.project_hdf5.keys()
def values(self):
"""
List all values for all groups and nodes of the HDF5 file
Returns:
list: list of all values
"""
return self.project_hdf5.values()
def items(self):
"""
List all keys and values as items of all groups and nodes of the HDF5 file
Returns:
list: list of sets (key, value)
"""
return self.project_hdf5.items()
def groups(self):
"""
Filter HDF5 file by groups
Returns:
FileHDFio: an HDF5 file which is filtered by groups
"""
return self.project_hdf5.groups()
def nodes(self):
"""
Filter HDF5 file by nodes
Returns:
FileHDFio: an HDF5 file which is filtered by nodes
"""
return self.project_hdf5.nodes()
def __enter__(self):
"""
Compatibility function for the with statement
"""
return self.project_hdf5.__enter__()
def __exit__(self, exc_type, exc_val, exc_tb):
"""
Compatibility function for the with statement
"""
self.project_hdf5.__exit__(exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb)
def __setitem__(self, key, value):
"""
Store data inside the HDF5 file
Args:
key (str): key to store the data
value (pandas.DataFrame, pandas.Series, dict, list, float, int): basically any kind of data is supported
"""
self.project_hdf5.__setitem__(key, value)
def __delitem__(self, key):
"""
Delete item from the HDF5 file
Args:
key (str): key of the item to delete
"""
self.project_hdf5.__delitem__(key)
def __str__(self):
"""
Machine readable string representation
Returns:
str: list all nodes and groups as string
"""
return self.project_hdf5.__str__()
def __repr__(self):
"""
Human readable string representation
Returns:
str: list all nodes and groups as string
"""
return self.project_hdf5.__repr__()
def __del__(self):
"""
When the object is deleted the HDF5 file has to be closed
"""
try:
self.project_hdf5._store.close()
except AttributeError:
pass
def __getitem__(self, item):
"""
Get/ read data from the HDF5 file
Args:
item (str, slice): path to the data or key of the data object
Returns:
dict, list, float, int: data or data object
"""
if item in self.list_files():
file_name = posixpath.join(self.working_directory, "{}".format(item))
with open(file_name) as f:
return f.readlines()
return self.project_hdf5.__getitem__(item)
class JobPath(JobPathBase):
"""
The JobPath class is derived from the JobCore and is used as a lean version of the GenericJob class. Instead of
loading the full pyiron object the JobPath class only provides access to the HDF5 file, which should be enough
for most analysis.
Args:
db (DatabaseAccess): database object
job_id (int): Job ID - optional, but either a job ID or a database entry db_entry has to be provided.
db_entry (dict): database entry {"job":, "subjob":, "projectpath":, "project":, "hamilton":, "hamversion":,
"status":} and optional entries are {"id":, "masterid":, "parentid":}
user (str): current unix/linux/windows user who is running pyiron
Attributes:
.. attribute:: job_name
name of the job, which has to be unique within the project
.. attribute:: status
execution status of the job, can be one of the following [initialized, appended, created, submitted, running,
aborted, collect, suspended, refresh, busy, finished]
.. attribute:: job_id
unique id to identify the job in the pyiron database
.. attribute:: parent_id
job id of the predecessor job - the job which was executed before the current one in the current job series
.. attribute:: master_id
job id of the master job - a meta job which groups a series of jobs, which are executed either in parallel or in
serial.
.. attribute:: child_ids
list of child job ids - only meta jobs have child jobs - jobs which list the meta job as their master
.. attribute:: project
Project instance the jobs is located in
.. attribute:: project_hdf5
ProjectHDFio instance which points to the HDF5 file the job is stored in
.. attribute:: job_info_str
short string to describe the job by it is job_name and job ID - mainly used for logging
.. attribute:: working_directory
working directory of the job is executed in - outside the HDF5 file
.. attribute:: path
path to the job as a combination of absolute file system path and path within the HDF5 file.
.. attribute:: is_root
boolean if the HDF5 object is located at the root level of the HDF5 file
.. attribute:: is_open
boolean if the HDF5 file is currently opened - if an active file handler exists
.. attribute:: is_empty
boolean if the HDF5 file is empty
.. attribute:: base_name
name of the HDF5 file but without any file extension
.. attribute:: file_path
directory where the HDF5 file is located
.. attribute:: h5_path
path inside the HDF5 file - also stored as absolute path
"""
def __init__(self, db, job_id=None, db_entry=None, user=None):
if db_entry is None and db is not None:
db_entry = db.get_item_by_id(job_id)
if db_entry is None:
raise ValueError("job ID {0} does not exist!".format(job_id))
hdf5_file = db_entry["subjob"].split("/")[1] + ".h5"
if db_entry["projectpath"] is not None:
job_path = db_entry["projectpath"]
else:
job_path = ''
job_path += db_entry["project"] + hdf5_file + db_entry["subjob"]
super(JobPath, self).__init__(job_path=job_path)
if "hamilton" in db_entry.keys():
self.__name__ = db_entry["hamilton"]
if "hamversion" in db_entry.keys():
self.__version__ = db_entry["hamversion"]
if "id" in db_entry.keys():
self._job_id = db_entry["id"]
if "status" in db_entry.keys():
self._status = db_entry["status"]
if "masterid" in db_entry.keys():
self._master_id = db_entry["masterid"]
if "parentid" in db_entry.keys():
self._parent_id = db_entry["parentid"]