From 8f2aeb02d53a7248cc0f4eebd5038ddcd90bbe17 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 12 Dec 2025 09:02:49 -0700 Subject: [PATCH 01/13] add analysis id to validate parameters --- qiita_db/processing_job.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index 837c7e24b..6add0bbd4 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -1470,6 +1470,7 @@ def _complete_artifact_transformation(self, artifacts_data): validator_jobs = [] with qdb.sql_connection.TRN: cmd_id = self.command.id + parameters = self.parameters.values for out_name, a_data in artifacts_data.items(): # Correct the format of the filepaths parameter so we can # create a validate job @@ -1506,6 +1507,12 @@ def _complete_artifact_transformation(self, artifacts_data): # belong to the same analysis, so we can just ask the # first artifact for the analysis that it belongs to analysis = self.input_artifacts[0].analysis.id + elif "analysis" in parameters: + # if we made it this far in the if/elif block it means that + # we are dealing with a job that was generated to link study/template + # artifacts and an analysis; thus, using the analysis parameter from + # the job itself + analysis = parameters["analysis"] # Once the validate job completes, it needs to know if it has # been generated from a command (and how) or if it has been @@ -1521,11 +1528,10 @@ def _complete_artifact_transformation(self, artifacts_data): cmd_out_id = qdb.sql_connection.TRN.execute_fetchlast() naming_params = self.command.naming_order if naming_params: - params = self.parameters.values art_name = "%s %s" % ( out_name, " ".join( - [str(params[p]).split("/")[-1] for p in naming_params] + [str(parameters[p]).split("/")[-1] for p in naming_params] ), ) else: From ad2cba849b62f3f76aa3dbe00a299665e4150ebe Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 12 Dec 2025 11:28:19 -0700 Subject: [PATCH 02/13] adding artifacts to job creation --- qiita_db/processing_job.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index 6add0bbd4..2771dafd5 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -1226,7 +1226,11 @@ def release(self): pvals = job.parameters.values if "analysis" in pvals: cmd_out_id = None - analysis = qdb.analysis.Analysis(job.parameters.values["analysis"]) + analysis = qdb.analysis.Analysis(pvals["analysis"]) + if "artifacts" in pvals: + parents = [ + qdb.artifact.Artifact(aid) for aid in pvals["artifacs"] + ] else: cmd_out_id = provenance["cmd_out_id"] analysis = None From 4dda3127df97f75923404d10fecc5cc31164b5ac Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 12 Dec 2025 13:24:32 -0700 Subject: [PATCH 03/13] add analysis to release --- qiita_db/processing_job.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index 2771dafd5..968917d0b 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -1227,10 +1227,6 @@ def release(self): if "analysis" in pvals: cmd_out_id = None analysis = qdb.analysis.Analysis(pvals["analysis"]) - if "artifacts" in pvals: - parents = [ - qdb.artifact.Artifact(aid) for aid in pvals["artifacs"] - ] else: cmd_out_id = provenance["cmd_out_id"] analysis = None @@ -1241,8 +1237,15 @@ def release(self): params = job.parameters cmd_out_id = provenance["cmd_out_id"] name = provenance["name"] - analysis = None data_type = None + if "analysis" in pvals: + analysis = qdb.analysis.Analysis(pvals["analysis"]) + if "artifacts" in pvals: + parents = [ + qdb.artifact.Artifact(aid) for aid in pvals["artifacs"] + ] + else: + analysis = None # Create the artifact atype = a_info["artifact_type"] From 3d2a0f2696a34ed663f922f5fa168a01f038b32e Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 12 Dec 2025 14:07:58 -0700 Subject: [PATCH 04/13] mv up pvals --- qiita_db/processing_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index 968917d0b..e45b0a9c5 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -1217,13 +1217,13 @@ def release(self): provenance = loads(self.parameters.values["provenance"]) job = ProcessingJob(provenance["job"]) + pvals = job.parameters.values if "data_type" in a_info: # This job is resulting from a private job parents = None params = None name = None data_type = a_info["data_type"] - pvals = job.parameters.values if "analysis" in pvals: cmd_out_id = None analysis = qdb.analysis.Analysis(pvals["analysis"]) From a5ddd936f92a0f0179c9e0e5f2822f7b9ba43afa Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sat, 13 Dec 2025 07:44:55 -0700 Subject: [PATCH 05/13] artifacs -> artifacts --- qiita_db/processing_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index e45b0a9c5..7396bae96 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -1242,7 +1242,7 @@ def release(self): analysis = qdb.analysis.Analysis(pvals["analysis"]) if "artifacts" in pvals: parents = [ - qdb.artifact.Artifact(aid) for aid in pvals["artifacs"] + qdb.artifact.Artifact(aid) for aid in pvals["artifacts"] ] else: analysis = None From 14b5a19b4803fb7f87b0f0bc71b7c0e059dcb41b Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sat, 13 Dec 2025 08:04:20 -0700 Subject: [PATCH 06/13] rm counts --- qiita_db/artifact.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index 4ea95160a..910860151 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -353,19 +353,7 @@ def create( "at least one filepath is required." ) - # Check that the combination of parameters is correct - counts = ( - int(bool(parents or processing_parameters)) - + int(prep_template is not None) - + int(bool(analysis or data_type)) - ) - if counts != 1: - # More than one parameter has been provided - raise qdb.exceptions.QiitaDBArtifactCreationError( - "One and only one of parents, prep template or analysis must " - "be provided" - ) - elif bool(parents) != bool(processing_parameters): + if bool(parents) != bool(processing_parameters): # When provided, parents and processing parameters both should be # provided (this is effectively doing an XOR) raise qdb.exceptions.QiitaDBArtifactCreationError( From 50d21091b6710ba603f47a769a125ab43b155e71 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sat, 13 Dec 2025 11:01:37 -0700 Subject: [PATCH 07/13] rewrite scenarios --- qiita_db/artifact.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index 910860151..2015cb7f9 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -329,16 +329,6 @@ def create( qiita_db.artifact.Artifact A new instance of Artifact - Raises - ------ - QiitaDBArtifactCreationError - If `filepaths` is not provided - If both `parents` and `prep_template` are provided - If none of `parents` and `prep_template` are provided - If `parents` is provided but `processing_parameters` is not - If both `prep_template` and `processing_parameters` is provided - If not all the artifacts in `parents` belong to the same study - Notes ----- The visibility of the artifact is set by default to `sandbox` if @@ -353,18 +343,30 @@ def create( "at least one filepath is required." ) - if bool(parents) != bool(processing_parameters): - # When provided, parents and processing parameters both should be - # provided (this is effectively doing an XOR) + # Check that the combination of parameters is correct + if bool(parents) and bool(prep_template): raise qdb.exceptions.QiitaDBArtifactCreationError( - "When provided, both parents and processing parameters should " - "be provided" + "Not valid: parents and prep_template provided" + ) + elif bool(parents) and not bool(processing_parameters): + raise qdb.exceptions.QiitaDBArtifactCreationError( + "Not valid: both parents and processing_parameters need to be provided" + ) + elif bool(prep_template) and bool(processing_parameters): + raise qdb.exceptions.QiitaDBArtifactCreationError( + "Not valid: both prep_template and processing_parameters provided" + ) + elif not bool(parents) and not bool(prep_template): + raise qdb.exceptions.QiitaDBArtifactCreationError( + "Not valid: both parents and prep_template need to be provided" ) elif bool(analysis) and not bool(data_type): - # When provided, analysis and data_type both should be - # provided (this is effectively doing an XOR) raise qdb.exceptions.QiitaDBArtifactCreationError( - "When provided, both analysis and data_type should be provided" + "Not valid: both analysis and data_type need to be provided" + ) + elif bool(prep_template) and bool(data_type): + raise qdb.exceptions.QiitaDBArtifactCreationError( + "Not valid: both prep_template and data_type need to be provided" ) # There are three different ways of creating an Artifact, but all of From 5a2cab2f499e304844decb8be90d4237be162269 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sat, 13 Dec 2025 11:16:54 -0700 Subject: [PATCH 08/13] fix analysis error --- qiita_db/artifact.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index 2015cb7f9..ea0045cae 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -356,10 +356,6 @@ def create( raise qdb.exceptions.QiitaDBArtifactCreationError( "Not valid: both prep_template and processing_parameters provided" ) - elif not bool(parents) and not bool(prep_template): - raise qdb.exceptions.QiitaDBArtifactCreationError( - "Not valid: both parents and prep_template need to be provided" - ) elif bool(analysis) and not bool(data_type): raise qdb.exceptions.QiitaDBArtifactCreationError( "Not valid: both analysis and data_type need to be provided" From 6a894958562214a781a07c9c57c7a84f0a10f6c7 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sat, 13 Dec 2025 15:07:15 -0700 Subject: [PATCH 09/13] no prep_template, parents or analysis provided --- qiita_db/artifact.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index ea0045cae..f4918bf07 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -364,6 +364,10 @@ def create( raise qdb.exceptions.QiitaDBArtifactCreationError( "Not valid: both prep_template and data_type need to be provided" ) + elif not bool(prep_template) and not bool(parents) and not bool(analysis): + raise qdb.exceptions.QiitaDBArtifactCreationError( + "Not valid: no prep_template, parents or analysis provided" + ) # There are three different ways of creating an Artifact, but all of # them execute a set of common operations. Declare functions to avoid From d9821710108d643c0807f6c0c5453751d738ed20 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sat, 13 Dec 2025 18:12:32 -0700 Subject: [PATCH 10/13] data_type --- qiita_db/artifact.py | 44 +++++++++++++++++++++++--------------- qiita_db/processing_job.py | 9 ++++++++ 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index f4918bf07..4ea95160a 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -329,6 +329,16 @@ def create( qiita_db.artifact.Artifact A new instance of Artifact + Raises + ------ + QiitaDBArtifactCreationError + If `filepaths` is not provided + If both `parents` and `prep_template` are provided + If none of `parents` and `prep_template` are provided + If `parents` is provided but `processing_parameters` is not + If both `prep_template` and `processing_parameters` is provided + If not all the artifacts in `parents` belong to the same study + Notes ----- The visibility of the artifact is set by default to `sandbox` if @@ -344,29 +354,29 @@ def create( ) # Check that the combination of parameters is correct - if bool(parents) and bool(prep_template): - raise qdb.exceptions.QiitaDBArtifactCreationError( - "Not valid: parents and prep_template provided" - ) - elif bool(parents) and not bool(processing_parameters): + counts = ( + int(bool(parents or processing_parameters)) + + int(prep_template is not None) + + int(bool(analysis or data_type)) + ) + if counts != 1: + # More than one parameter has been provided raise qdb.exceptions.QiitaDBArtifactCreationError( - "Not valid: both parents and processing_parameters need to be provided" + "One and only one of parents, prep template or analysis must " + "be provided" ) - elif bool(prep_template) and bool(processing_parameters): + elif bool(parents) != bool(processing_parameters): + # When provided, parents and processing parameters both should be + # provided (this is effectively doing an XOR) raise qdb.exceptions.QiitaDBArtifactCreationError( - "Not valid: both prep_template and processing_parameters provided" + "When provided, both parents and processing parameters should " + "be provided" ) elif bool(analysis) and not bool(data_type): + # When provided, analysis and data_type both should be + # provided (this is effectively doing an XOR) raise qdb.exceptions.QiitaDBArtifactCreationError( - "Not valid: both analysis and data_type need to be provided" - ) - elif bool(prep_template) and bool(data_type): - raise qdb.exceptions.QiitaDBArtifactCreationError( - "Not valid: both prep_template and data_type need to be provided" - ) - elif not bool(prep_template) and not bool(parents) and not bool(analysis): - raise qdb.exceptions.QiitaDBArtifactCreationError( - "Not valid: no prep_template, parents or analysis provided" + "When provided, both analysis and data_type should be provided" ) # There are three different ways of creating an Artifact, but all of diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index 7396bae96..c64e82060 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -1244,6 +1244,15 @@ def release(self): parents = [ qdb.artifact.Artifact(aid) for aid in pvals["artifacts"] ] + # as this is going to be the first artifact of an analysis, we + # need to provide the data type so we are going to make sure all + # the parents data_types are the same and assing that one + data_type = set([p.data_type for p in parents]) + if len(data_type) != 1: + raise ValueError( + f"Not valida parents data_types: {data_type}" + ) + data_type = data_type.pop() else: analysis = None From fdff9042a89cd5a22d539a984aa208e19d0dcbb3 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sun, 14 Dec 2025 07:35:16 -0700 Subject: [PATCH 11/13] rm parents --- qiita_db/processing_job.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index c64e82060..93cc4f0e9 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -1241,13 +1241,15 @@ def release(self): if "analysis" in pvals: analysis = qdb.analysis.Analysis(pvals["analysis"]) if "artifacts" in pvals: - parents = [ - qdb.artifact.Artifact(aid) for aid in pvals["artifacts"] - ] # as this is going to be the first artifact of an analysis, we # need to provide the data type so we are going to make sure all # the parents data_types are the same and assing that one - data_type = set([p.data_type for p in parents]) + data_type = set( + [ + qdb.artifact.Artifact(aid).data_type + for aid in pvals["artifacts"] + ] + ) if len(data_type) != 1: raise ValueError( f"Not valida parents data_types: {data_type}" From 44eeaf5790ac63690fbbe573be5d25fe49aec2f4 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sun, 14 Dec 2025 16:06:56 -0700 Subject: [PATCH 12/13] params=None --- qiita_db/processing_job.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index 93cc4f0e9..122ee4aee 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -1243,7 +1243,12 @@ def release(self): if "artifacts" in pvals: # as this is going to be the first artifact of an analysis, we # need to provide the data type so we are going to make sure all - # the parents data_types are the same and assing that one + # the parents data_types are the same and assigning that one; note + # that (1) we are doing this to be stringent but it should be responsability + # of the plugin creating this artifact, and (2) to keep the same functionality + # we are going to make sure that params is not set as it shouldn't be passed when + # assiging to an analysis + params = None data_type = set( [ qdb.artifact.Artifact(aid).data_type From 89a2916e01d2cc062d34be4b206e36bafd3e249a Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 15 Dec 2025 18:08:54 -0700 Subject: [PATCH 13/13] overwrite_lock --- qiita_db/analysis.py | 8 ++++++-- qiita_db/processing_job.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/qiita_db/analysis.py b/qiita_db/analysis.py index 44f37e365..2d80d5e4d 100644 --- a/qiita_db/analysis.py +++ b/qiita_db/analysis.py @@ -847,7 +847,7 @@ def _lock_samples(self): "Can't add/remove samples from this analysis" ) - def add_samples(self, samples): + def add_samples(self, samples, overwrite_lock=False): """Adds samples to the analysis Parameters @@ -855,9 +855,13 @@ def add_samples(self, samples): samples : dictionary of lists samples and the artifact id they come from in form {artifact_id: [sample1, sample2, ...], ...} + overwrite_lock : bool, optional + if True it will ignore the sample-lock and will allow adding + samples to a non-default analysis """ with qdb.sql_connection.TRN: - self._lock_samples() + if not overwrite_lock: + self._lock_samples() for aid, samps in samples.items(): # get previously selected samples for aid and filter them out diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index 122ee4aee..ec4985bbb 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -1257,7 +1257,7 @@ def release(self): ) if len(data_type) != 1: raise ValueError( - f"Not valida parents data_types: {data_type}" + f"Not valid parents data_types: {data_type}" ) data_type = data_type.pop() else: