Skip to content

Commit

Permalink
'save dependency level in database'
Browse files Browse the repository at this point in the history
  • Loading branch information
@LStruber
LStruber committed Mar 16, 2022
1 parent 290f9ce commit ca7fc20
Show file tree
Hide file tree
Showing 5 changed files with 68 additions and 50 deletions.
47 changes: 27 additions & 20 deletions python/populse_mia/data_manager/data_history_inspect.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
BRICK_OUTPUTS, COLLECTION_BRICK,
COLLECTION_CURRENT,
COLLECTION_INITIAL, TAG_BRICKS,
TAG_BRICKS_DEP_LVL,
TAG_CHECKSUM, TAG_EXP_TYPE,
TAG_FILENAME, TAG_TYPE, TYPE_MAT,
TYPE_NII, TYPE_TXT, TYPE_UNKNOWN)
Expand Down Expand Up @@ -201,7 +202,7 @@ def get_data_history_in_brick_tag(filename, project):
'''

session = project.session

bricks_dep_list = session.get_value(COLLECTION_CURRENT, filename, TAG_BRICKS_DEP_LVL)
procs, proc_list = get_procs_in_brick_tag(filename, project)
links = set()

Expand All @@ -225,6 +226,7 @@ def get_data_history_in_brick_tag(filename, project):

while proc_list:
proc = proc_list.pop(0)

if proc in done_procs:
continue
done_procs.add(proc)
Expand All @@ -242,32 +244,37 @@ def get_data_history_in_brick_tag(filename, project):

for name, (value, filenames) in values_w_files.items():
for nfilename in filenames:
exact_links_found = False
other_links_found = {}
for prev_proc in proc_list:
links_found = []
links_dep_level = []
for i in range(0, len(proc_list)):
prev_proc = proc_list[i]
dep_level = bricks_dep_list[i]

# connect outputs of prev_proc which are identical to
if prev_proc != proc:
print('look for value', value, 'in', prev_proc.brick[BRICK_NAME])

for pname, pval in prev_proc.brick[BRICK_OUTPUTS].items():
if pval == value:
exact_links_found = True
links.add((prev_proc, pname, proc, name))
links_found.append((prev_proc, pname, 'exact'))
links_dep_level.append(dep_level)
elif data_in_value(pval, nfilename, project):
other_links_found[pname] = pval

# treat "other" links only if no exact links were found
# for example, fix issue of list duplicate redundant outputs
if other_links_found and not exact_links_found:
for pname, pval in other_links_found.items():
links.add((prev_proc, pname, proc, name))

if exact_links_found or other_links_found:
break

if not exact_links_found and not other_links_found:
# the param has no previous processing or just the current
# self-modifing process: connect it to main inputs
links_found.append((prev_proc, pname, 'other'))
links_dep_level.append(dep_level)

if links_found:
min_dep_level = min(links_dep_level)
indices = [i for i, v in enumerate(links_dep_level) if v == min_dep_level]
exact_link = False
for i in indices:
if links_found[i][2] == 'exact':
exact_link = True
links.add((links_found[i][0], links_found[i][1], proc, name))
if not exact_link:
for i in indices:
if links_found[i][2] == 'other':
links.add((links_found[i][0], links_found[i][1], proc, name))
else:
links.add((None, name, proc, name))

print('history of:', filename, ':', len([p for p in procs.values() if p.used]), 'processes, ', len(links), 'links')
Expand Down
9 changes: 8 additions & 1 deletion python/populse_mia/data_manager/project.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
from populse_db.database import (
FIELD_TYPE_STRING, FIELD_TYPE_LIST_STRING,
FIELD_TYPE_JSON, FIELD_TYPE_DATETIME,
FIELD_TYPE_INTEGER)
FIELD_TYPE_INTEGER, FIELD_TYPE_LIST_INTEGER)

from capsul.api import Pipeline
from capsul.pipeline import pipeline_tools
Expand All @@ -50,6 +50,7 @@
TAG_EXP_TYPE = "Exp Type"
TAG_FILENAME = "FileName"
TAG_BRICKS = "Bricks"
TAG_BRICKS_DEP_LVL = "Level of dependency"
CLINICAL_TAGS = ["Site", "Spectro", "MR", "PatientRef", "Pathology", "Age",
"Sex", "Message"]
BRICK_ID = "ID"
Expand Down Expand Up @@ -228,6 +229,12 @@ def __init__(self, project_root_folder, new_project):
self.session.add_field(COLLECTION_INITIAL, TAG_BRICKS,
FIELD_TYPE_LIST_STRING, "Path bricks", True,
TAG_ORIGIN_BUILTIN, None, None)
self.session.add_field(COLLECTION_CURRENT, TAG_BRICKS_DEP_LVL,
FIELD_TYPE_LIST_INTEGER, "Level of dependency", True,
TAG_ORIGIN_BUILTIN, None, None)
self.session.add_field(COLLECTION_INITIAL, TAG_BRICKS_DEP_LVL,
FIELD_TYPE_LIST_INTEGER, "Level of dependency", True,
TAG_ORIGIN_BUILTIN, None, None)

self.session.add_field(COLLECTION_BRICK, BRICK_NAME,
FIELD_TYPE_STRING, "Brick name", False,
Expand Down
16 changes: 8 additions & 8 deletions python/populse_mia/user_interface/data_browser/data_browser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
check_value_type, set_item_data, table_to_database)
from populse_mia.data_manager.project import (
COLLECTION_CURRENT, COLLECTION_INITIAL, COLLECTION_BRICK, TAG_CHECKSUM,
TAG_FILENAME, TAG_BRICKS, BRICK_NAME)
TAG_FILENAME, TAG_BRICKS, TAG_BRICKS_DEP_LVL, BRICK_NAME)
from populse_mia.data_manager.database_mia import (
TAG_ORIGIN_BUILTIN, TAG_ORIGIN_USER)
from populse_mia.software_properties import Config
Expand Down Expand Up @@ -843,6 +843,7 @@ def add_columns(self):
tags = self.project.session.get_fields_names(COLLECTION_CURRENT)
tags.remove(TAG_CHECKSUM)
tags.remove(TAG_FILENAME)
tags.remove(TAG_BRICKS_DEP_LVL)
tags = sorted(tags)
tags.insert(0, TAG_FILENAME)

Expand Down Expand Up @@ -990,6 +991,7 @@ def add_rows(self, rows):
else:
cur_value = self.project.session.get_value(
COLLECTION_CURRENT, scan, tag)

if cur_value is not None:
if tag != TAG_BRICKS:
set_item_data(
Expand Down Expand Up @@ -1027,9 +1029,10 @@ def add_rows(self, rows):
brick_name_button.moveToThread(
QApplication.instance().thread())
self.bricks[brick_name_button] = brick_uuid
brick_name_button.clicked.connect(
partial(self.show_brick_history,
scan['FileName']))
if 'FileName' in scan:
brick_name_button.clicked.connect(
partial(self.show_brick_history,
scan['FileName']))
layout.addWidget(brick_name_button)
# End of change for issue#236 (2022-03-15)

Expand All @@ -1051,7 +1054,6 @@ def add_rows(self, rows):
item.flags() & ~Qt.ItemIsEditable)
self.setItem(rowCount, column, item)


# Crash if self.setSortingEnabled(True) because it calls sortByColumn()
# self.setSortingEnabled(False)

Expand Down Expand Up @@ -1460,7 +1462,6 @@ def fill_cells_update_table(self):

for scan in scans:
for column, current_tag in enumerate(tags):

idx += 1
self.progress.setValue(idx)
QApplication.processEvents()
Expand Down Expand Up @@ -1543,8 +1544,6 @@ def fill_cells_update_table(self):

self.setItem(row, column, item)



row += 1

# We apply the saved sort when the project is opened or after the
Expand Down Expand Up @@ -1581,6 +1580,7 @@ def fill_headers(self, take_tags_to_update=False):
tags = self.project.session.get_fields_names(COLLECTION_CURRENT)
tags.remove(TAG_CHECKSUM)
tags.remove(TAG_FILENAME)
tags.remove(TAG_BRICKS_DEP_LVL)
tags = sorted(tags)
tags.insert(0, TAG_FILENAME)

Expand Down
30 changes: 15 additions & 15 deletions python/populse_mia/user_interface/pipeline_manager/pipeline_manager_tab.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
BRICK_OUTPUTS, COLLECTION_BRICK,
COLLECTION_CURRENT,
COLLECTION_INITIAL, TAG_BRICKS,
TAG_CHECKSUM, TAG_EXP_TYPE,
TAG_CHECKSUM, TAG_BRICKS_DEP_LVL,
TAG_FILENAME, TAG_TYPE, TYPE_MAT,
TYPE_NII, TYPE_TXT, TYPE_UNKNOWN)
from populse_mia.software_properties import Config
Expand Down Expand Up @@ -521,18 +521,14 @@ def add_plug_value_to_database(self, p_value, brick_id, brick_history, node_name

# Adding the new brick to the output files
bricks = []
brick_dep_lvl = []
# if brick_id not in bricks:
bricks.append(brick_id)
for brid in brick_history:
brick_dep_lvl.append(0)
for brick in brick_history:
# if brid not in bricks:
bricks.append(brid)

# old_bricks = self.project.session.get_value(
# COLLECTION_CURRENT, p_value, TAG_BRICKS)
# if old_bricks is not None:
# for brid in old_bricks:
# # if brid not in bricks:
# bricks.append(brid)
bricks.append(brick[0])
brick_dep_lvl.append(brick[1])

# Type tag
filename, file_extension = os.path.splitext(p_value)
Expand Down Expand Up @@ -608,7 +604,8 @@ def add_plug_value_to_database(self, p_value, brick_id, brick_history, node_name
#banished_tags = set([TAG_TYPE, TAG_EXP_TYPE, TAG_BRICKS,
# TAG_CHECKSUM, TAG_FILENAME])
banished_tags = set([TAG_TYPE, TAG_BRICKS,
TAG_CHECKSUM, TAG_FILENAME])
TAG_CHECKSUM, TAG_FILENAME,
TAG_BRICKS_DEP_LVL])
cvalues = {field: getattr(scan, field)
for field in field_names
if field not in banished_tags}
Expand Down Expand Up @@ -705,9 +702,11 @@ def add_plug_value_to_database(self, p_value, brick_id, brick_history, node_name
all_ivalues = {pop_up.key: all_ivalues[pop_up.key]}

cvalues = {TAG_TYPE: ptype,
TAG_BRICKS: bricks}
TAG_BRICKS: bricks,
TAG_BRICKS_DEP_LVL: brick_dep_lvl}
ivalues = {TAG_TYPE: ptype,
TAG_BRICKS: bricks}
TAG_BRICKS: bricks,
TAG_BRICKS_DEP_LVL: brick_dep_lvl}

# from here if we still have several tags sets, we do not assign them
# at all. Otherwise, set them.
Expand Down Expand Up @@ -2610,6 +2609,7 @@ def update_auto_inheritance(self, job, node):
def update_job_history(self, job):
dependencies_list = []
for dep in self.workflow.dependencies:
dependency_level = 1
if dep[1] is job:
if hasattr(dep[0], 'process'):
node = dep[0].process()
Expand All @@ -2622,11 +2622,11 @@ def update_job_history(self, job):
# Adding the brick to the bricks history
if not isinstance(node, (PipelineNode, Pipeline)):
# if dep[0] not in dependencies_list:
dependencies_list.append(dep[0].uuid)
dependencies_list.append((dep[0].uuid, dependency_level))
sub_dependencies_list = self.update_job_history(dep[0])
for sub_dep in sub_dependencies_list:
# if sub_dep not in dependencies_list:
dependencies_list.append(sub_dep)
dependencies_list.append((sub_dep[0], dependency_level + sub_dep[1]))
return dependencies_list

def update_inheritance(self, job, node):
Expand Down
16 changes: 10 additions & 6 deletions python/populse_mia/user_interface/pop_ups.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@
BRICK_EXEC, BRICK_EXEC_TIME, BRICK_INIT, BRICK_INIT_TIME,
BRICK_INPUTS, BRICK_NAME, BRICK_OUTPUTS, COLLECTION_BRICK,
COLLECTION_CURRENT, COLLECTION_INITIAL, Project, TAG_CHECKSUM, TAG_FILENAME,
TAG_TYPE, TYPE_MAT, TYPE_NII, TYPE_TXT, TYPE_UNKNOWN)
TAG_TYPE, TAG_BRICKS_DEP_LVL, TYPE_MAT, TYPE_NII, TYPE_TXT, TYPE_UNKNOWN)
from populse_mia.software_properties import Config, verCmp
from populse_mia.user_interface.data_browser import data_browser
from populse_mia.utils import utils
Expand Down Expand Up @@ -857,6 +857,8 @@ def __init__(self, databrowser, project):

tags_lists = project.session.get_fields_names(COLLECTION_CURRENT)
tags_lists.remove(TAG_CHECKSUM)
tags_lists.remove(TAG_BRICKS_DEP_LVL)

for tag in tags_lists:
item = QtWidgets.QListWidgetItem()
self.list_widget_tags.addItem(item)
Expand Down Expand Up @@ -928,6 +930,8 @@ def search_str(self, project, str_search):
return_list = []
tags_lists = project.session.get_fields_names(COLLECTION_CURRENT)
tags_lists.remove(TAG_CHECKSUM)
tags_lists.remove(TAG_BRICKS_DEP_LVL)

if str_search != "":
for tag in tags_lists:
if str_search.upper() in tag.upper():
Expand Down Expand Up @@ -4611,13 +4615,13 @@ def search_str(self, str_search):
if str_search != "":
for tag in self.project.session.get_fields_names(
COLLECTION_CURRENT):
if tag != TAG_CHECKSUM:
if tag != TAG_CHECKSUM and tag != TAG_BRICKS_DEP_LVL:
if str_search.upper() in tag.upper():
return_list.append(tag)
else:
for tag in self.project.session.get_fields_names(
COLLECTION_CURRENT):
if tag != TAG_CHECKSUM:
if tag != TAG_CHECKSUM and tag != TAG_BRICKS_DEP_LVL:
return_list.append(tag)

for idx in range(self.list_widget_tags.count()):
Expand Down Expand Up @@ -4650,7 +4654,7 @@ def __init__(self, project):

# Filling the list and checking the thumbnail tag
for tag in self.project.session.get_fields_names(COLLECTION_CURRENT):
if tag != TAG_CHECKSUM:
if tag != TAG_CHECKSUM and tag != TAG_BRICKS_DEP_LVL:
item = QtWidgets.QListWidgetItem()
item.setFlags(item.flags() | QtCore.Qt.ItemIsUserCheckable)
if tag == self.config.getThumbnailTag():
Expand Down Expand Up @@ -4697,7 +4701,7 @@ def __init__(self, project, tags_to_display, tag_name_checked=None):

self.selected_tag = None
for tag in tags_to_display:
if tag != TAG_CHECKSUM:
if tag != TAG_CHECKSUM and tag != TAG_BRICKS_DEP_LVL:
item = QtWidgets.QListWidgetItem()
item.setFlags(item.flags() | QtCore.Qt.ItemIsUserCheckable)
if tag == tag_name_checked:
Expand Down Expand Up @@ -5029,7 +5033,7 @@ def __init__(self, project, visualized_tags):
# the tags on the left (invisible tags)

for tag in project.session.get_fields_names(COLLECTION_CURRENT):
if tag != TAG_CHECKSUM and tag != TAG_FILENAME:
if tag != TAG_CHECKSUM and tag != TAG_FILENAME and tag != TAG_BRICKS_DEP_LVL:
item = QtWidgets.QListWidgetItem()
if tag not in self.visualized_tags:
# Tag not visible: left side
Expand Down

0 comments on commit ca7fc20

Please sign in to comment.