From db968c3b4b8b52cf97473ca6de34655a96b92b83 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 10 Dec 2024 13:25:28 -0700 Subject: [PATCH 1/9] some changes --- qiita_db/metadata_template/prep_template.py | 16 +++++++++++++- qiita_db/processing_job.py | 12 ++++++----- qiita_db/support_files/patches/93.sql | 7 ++++++ qiita_db/util.py | 14 ++++++++---- qiita_pet/templates/resources.html | 24 +++++++++------------ scripts/qiita-recover-jobs | 3 ++- 6 files changed, 51 insertions(+), 25 deletions(-) diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index 52f781b9a..3ae26d17d 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -11,6 +11,7 @@ from iteration_utilities import duplicates from qiita_core.exceptions import IncompetentQiitaDeveloperError +from qiita_ware.private_plugin import _delete_analysis_artifacts import qiita_db as qdb from .constants import (PREP_TEMPLATE_COLUMNS, TARGET_GENE_DATA_TYPES, PREP_TEMPLATE_COLUMNS_TARGET_GENE) @@ -135,7 +136,7 @@ def create(cls, md_template, study, data_type, investigation_type=None, # data_type being created - if possible if investigation_type is None: if data_type_str in TARGET_GENE_DATA_TYPES: - investigation_type = 'Amplicon' + investigation_type = 'AMPLICON' elif data_type_str == 'Metagenomic': investigation_type = 'WGS' elif data_type_str == 'Metatranscriptomic': @@ -282,6 +283,19 @@ def delete(cls, id_): qdb.sql_connection.TRN.execute_fetchflatten()) if archived_artifacts: for aid in archived_artifacts: + # before we can delete the archived artifact, we need + # to delete the analyses where they were used. + sql = """SELECT analysis_id + FROM qiita.analysis + WHERE analysis_id IN ( + SELECT DISTINCT analysis_id + FROM qiita.analysis_sample + WHERE artifact_id IN %s)""" + qdb.sql_connection.TRN.add(sql, [aid]) + analyses = set( + qdb.sql_connection.TRN.execute_fetchflatten()) + for _id in analyses: + _delete_analysis_artifacts(qdb.analysis.Analysis(_id)) qdb.artifact.Artifact.delete(aid) # Delete the prep template filepaths diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index a8844d181..27192bab7 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -2053,23 +2053,25 @@ def complete_processing_job(self): def trace(self): """ Returns as a text array the full trace of the job, from itself to validators and complete jobs""" - lines = [f'{self.id} [{self.external_id}]: ' + lines = [f'{self.id} [{self.external_id}] ({self.status}): ' f'{self.command.name} | {self.resource_allocation_info}'] cjob = self.complete_processing_job if cjob is not None: - lines.append(f' {cjob.id} [{cjob.external_id}] | ' + lines.append(f' {cjob.id} [{cjob.external_id}] ({cjob.status})| ' f'{cjob.resource_allocation_info}') vjob = self.release_validator_job if vjob is not None: lines.append(f' {vjob.id} [{vjob.external_id}] ' - f'| {vjob.resource_allocation_info}') + f' ({vjob.status}) | ' + f'{vjob.resource_allocation_info}') for v in self.validator_jobs: - lines.append(f' {v.id} [{v.external_id}]: ' + lines.append(f' {v.id} [{v.external_id}] ({v.status}): ' f'{v.command.name} | {v.resource_allocation_info}') cjob = v.complete_processing_job if cjob is not None: lines.append(f' {cjob.id} [{cjob.external_id}] ' - f'| {cjob.resource_allocation_info}') + f'({cjob.status}) | ' + f'{cjob.resource_allocation_info}') return lines diff --git a/qiita_db/support_files/patches/93.sql b/qiita_db/support_files/patches/93.sql index 81abc3331..4befc74d6 100644 --- a/qiita_db/support_files/patches/93.sql +++ b/qiita_db/support_files/patches/93.sql @@ -55,3 +55,10 @@ CREATE INDEX IF NOT EXISTS processing_job_command_parameters_payload ON qiita.pr -- After the changes -- 18710.404 ms + +-- + +-- Nov 5, 2024 +-- Addding contraints for the slurm_reservation column +ALTER TABLE qiita.analysis DROP CONSTRAINT IF EXISTS analysis_slurm_reservation_valid_chars; +ALTER TABLE qiita.analysis ADD CONSTRAINT analysis_slurm_reservation_valid_chars CHECK ( slurm_reservation ~ '^[a-zA-Z0-9_]*$' ); diff --git a/qiita_db/util.py b/qiita_db/util.py index 0dae03431..30731fa8b 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -2334,7 +2334,7 @@ def send_email(to, subject, body): msg = MIMEMultipart() msg['From'] = qiita_config.smtp_email msg['To'] = to - msg['Subject'] = subject + msg['Subject'] = subject.replace('\n', '') msg.attach(MIMEText(body, 'plain')) # connect to smtp server, using ssl if needed @@ -2496,9 +2496,9 @@ def _resource_allocation_plot_helper( ax.set_ylabel(curr) ax.set_xlabel(col_name) - # 100 - number of maximum iterations, 3 - number of failures we tolerate + # 50 - number of maximum iterations, 3 - number of failures we tolerate best_model, options = _resource_allocation_calculate( - df, x_data, y_data, models, curr, col_name, 100, 3) + df, x_data, y_data, models, curr, col_name, 50, 3) k, a, b = options.x x_plot = np.array(sorted(df[col_name].unique())) y_plot = best_model(x_plot, k, a, b) @@ -2593,6 +2593,8 @@ def _resource_allocation_calculate( failures_df = _resource_allocation_failures( df, k, a, b, model, col_name, type_) y_plot = model(x, k, a, b) + if not any(y_plot): + continue cmax = max(y_plot) cmin = min(y_plot) failures = failures_df.shape[0] @@ -2834,13 +2836,17 @@ def merge_rows(rows): wait_time = ( datetime.strptime(rows.iloc[0]['Start'], date_fmt) - datetime.strptime(rows.iloc[0]['Submit'], date_fmt)) - tmp = rows.iloc[1].copy() + if rows.shape[0] >= 2: + tmp = rows.iloc[1].copy() + else: + tmp = rows.iloc[0].copy() tmp['WaitTime'] = wait_time return tmp slurm_data['external_id'] = slurm_data['JobID'].apply( lambda x: int(x.split('.')[0])) slurm_data['external_id'] = slurm_data['external_id'].ffill() + slurm_data = slurm_data.groupby( 'external_id').apply(merge_rows).reset_index(drop=True) diff --git a/qiita_pet/templates/resources.html b/qiita_pet/templates/resources.html index d4b9e870c..e044429d2 100644 --- a/qiita_pet/templates/resources.html +++ b/qiita_pet/templates/resources.html @@ -21,14 +21,14 @@

Please choose software, version, and command to view the data.

- +
- +