From 86085d42d13f3a6aa63db405e806402ab66ae1da Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 14:58:19 +0000
Subject: [PATCH 01/33] update Makefile and requirements.txt

Unpinned SQLAlchemy so that Python 3.9 can be used.
Updated makefile to use -o and --run_id arguments in new CLI.
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 15d97e59..62841ab0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,5 +10,5 @@ Pillow
 PyQt5
 scipy
 seaborn
-sqlalchemy==1.3.10
+sqlalchemy
 tqdm

From b5ef8b5d4c817fa382615e527e65936acd9b7b00 Mon Sep 17 00:00:00 2001
From: baileythegreen <baileythegreen@gmail.com>
Date: Mon, 18 Oct 2021 20:05:19 +0100
Subject: [PATCH 02/33] Add `multiprocessing` support to `pyani plot`

---
 pyani/scripts/subcommands/subcmd_plot.py | 31 +++++++++++-------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/pyani/scripts/subcommands/subcmd_plot.py b/pyani/scripts/subcommands/subcmd_plot.py
index d3682d1d..8070ed42 100644
--- a/pyani/scripts/subcommands/subcmd_plot.py
+++ b/pyani/scripts/subcommands/subcmd_plot.py
@@ -117,6 +117,11 @@ def write_run_plots(run_id: int, session, outfmts: List[str], args: Namespace) -
     logger.debug(
         f"Have {len(result_label_dict)} labels and {len(result_class_dict)} classes"
     )
+    result_label_dict = pyani_orm.get_matrix_labels_for_run(session, args.run_id)
+    result_class_dict = pyani_orm.get_matrix_classes_for_run(session, args.run_id)
+    logger.debug(
+        f"Have {len(result_label_dict)} labels and {len(result_class_dict)} classes"
+    )
 
     # Write heatmap and distribution plot for each results matrix
 
@@ -141,24 +146,16 @@ def write_run_plots(run_id: int, session, outfmts: List[str], args: Namespace) -
                 [run_id, matdata, result_label_dict, result_class_dict, outfmts, args],
             )
         )
-        plotting_commands.append((write_distribution, [run_id, matdata, outfmts, args]))
-
-    id_matrix = MatrixData("identity", pd.read_json(results.df_identity), {})
-    cov_matrix = MatrixData("coverage", pd.read_json(results.df_coverage), {})
-    plotting_commands.append(
-        (
-            write_scatter,
-            [
-                run_id,
-                id_matrix,
-                cov_matrix,
-                result_label_dict,
-                result_class_dict,
-                outfmts,
-                args,
-            ],
+        plotting_commands.append(
+            (write_distribution, [run_id, matdata, outfmts, args])  # type: ignore
         )
-    )
+
+    # Run the plotting commands
+    [pool.apply_async(func, args, {}) for func, args in plotting_commands]
+
+    # Close worker pool
+    pool.close()
+    pool.join()
 
     # Run the plotting commands
     logger.debug("Running plotting commands")

From c5fc109d970fbd2a41d5896ee04ecf0a4af2042e Mon Sep 17 00:00:00 2001
From: Bailey Harrington <baileythegreen@gmail.com>
Date: Fri, 19 Nov 2021 17:06:15 +0000
Subject: [PATCH 03/33] Revert "Parser choices"

---
 pyani/pyani_tools.py                   | 13 ++++---------
 pyani/scripts/parsers/plot_parser.py   |  4 ++--
 pyani/scripts/parsers/report_parser.py |  4 ++--
 3 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/pyani/pyani_tools.py b/pyani/pyani_tools.py
index a9b93ae8..ae5ab8ad 100644
--- a/pyani/pyani_tools.py
+++ b/pyani/pyani_tools.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # (c) The James Hutton Institute 2013-2019
-# (c) The University of Strathclude 2019-2020
+# (c) The University of Strathclude 2019-2022
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -19,7 +19,7 @@
 # The MIT License
 #
 # Copyright (c) 2013-2019 The James Hutton Institute
-# (c) The University of Strathclude 2019-2020
+# (c) The University of Strathclude 2019-2022
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -303,13 +303,8 @@ def label_results_matrix(matrix: pd.DataFrame, labels: Dict) -> pd.DataFrame:
     Applies the labels from the dictionary to the dataframe in
     matrix, and returns the result.
     """
-    # The dictionary uses string keys!
-    # Create a label function that produces <label>:<genome_id>
-    # when a label is available; and just Genome_id:<genome_id> when no
-    # label exists
-    label = lambda gen_id: f"{labels.get(str(gen_id), 'Genome_id')}:{gen_id}"
-    matrix.columns = [label(_) for _ in matrix.columns]
-    matrix.index = [label(_) for _ in matrix.index]
+    matrix.columns = [f"{labels.get(_, _)}:{_}" for _ in matrix.columns]
+    matrix.index = [f"{labels.get(_, _)}:{_}" for _ in matrix.index]
     return matrix
 
 
diff --git a/pyani/scripts/parsers/plot_parser.py b/pyani/scripts/parsers/plot_parser.py
index 0aac8031..b35f8e01 100644
--- a/pyani/scripts/parsers/plot_parser.py
+++ b/pyani/scripts/parsers/plot_parser.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # (c) The James Hutton Institute 2016-2019
-# (c) University of Strathclyde 2019-2020
+# (c) University of Strathclyde 2019-2022
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -17,7 +17,7 @@
 # The MIT License
 #
 # Copyright (c) 2016-2019 The James Hutton Institute
-# Copyright (c) 2019-2020 University of Strathclyde
+# Copyright (c) 2019-2022 University of Strathclyde
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
diff --git a/pyani/scripts/parsers/report_parser.py b/pyani/scripts/parsers/report_parser.py
index 22767a79..c1ca01e1 100644
--- a/pyani/scripts/parsers/report_parser.py
+++ b/pyani/scripts/parsers/report_parser.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # (c) The James Hutton Institute 2016-2019
-# (c) University of Strathclyde 2019-2020
+# (c) University of Strathclyde 2019-2022
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -17,7 +17,7 @@
 # The MIT License
 #
 # Copyright (c) 2016-2019 The James Hutton Institute
-# Copyright (c) 2019-2020 University of Strathclyde
+# Copyright (c) 2019-2022 University of Strathclyde
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

From da644da6391c6a5d6dac4299375c207fc30eafa3 Mon Sep 17 00:00:00 2001
From: baileythegreen <baileythegreen@gmail.com>
Date: Mon, 29 Nov 2021 15:26:42 +0000
Subject: [PATCH 04/33] Add parser option to override matrix labels

---
 pyani/scripts/subcommands/subcmd_report.py | 27 +++++-----------------
 1 file changed, 6 insertions(+), 21 deletions(-)

diff --git a/pyani/scripts/subcommands/subcmd_report.py b/pyani/scripts/subcommands/subcmd_report.py
index 82423893..81d65f88 100644
--- a/pyani/scripts/subcommands/subcmd_report.py
+++ b/pyani/scripts/subcommands/subcmd_report.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # (c) The James Hutton Institute 2017-2019
-# (c) University of Strathclyde 2019-2020
+# (c) University of Strathclyde 2019-2022
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -18,7 +18,7 @@
 # The MIT License
 #
 # Copyright (c) 2017-2019 The James Hutton Institute
-# Copyright (c) 2019-2020 University of Strathclyde
+# Copyright (c) 2019-2022 University of Strathclyde
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -154,12 +154,7 @@ def subcmd_report(args: Namespace) -> int:
             "genome label",
             "genome class",
         ]
-        report(
-            args,
-            session,
-            formats,
-            ReportParams("runs_genomes", statement, headers),
-        )
+        report(args, session, formats, ReportParams("runs_genomes", statement, headers))
 
     # Report table of all runs in which a genome is involved
     if args.show_genomes_runs:
@@ -196,12 +191,7 @@ def subcmd_report(args: Namespace) -> int:
             "method",
             "date run",
         ]
-        report(
-            args,
-            session,
-            formats,
-            ReportParams("genomes_runs", statement, headers),
-        )
+        report(args, session, formats, ReportParams("genomes_runs", statement, headers))
 
     # Report table of comparison results for the indicated runs
     if args.run_results:
@@ -266,7 +256,7 @@ def subcmd_report(args: Namespace) -> int:
     # our matrices directly, here
     if args.run_matrices:
         show_index = not args.no_matrix_labels
-        for run_id in [run_id for run_id in args.run_matrices]:
+        for run_id in [run_id.strip() for run_id in args.run_matrices.split(",")]:
             logger.debug("Extracting matrices for run %s", run_id)
             run = session.query(Run).filter(Run.run_id == run_id).first()
             matlabel_dict = get_matrix_labels_for_run(session, run_id)
@@ -300,12 +290,7 @@ def subcmd_report(args: Namespace) -> int:
     return 0
 
 
-def report(
-    args: Namespace,
-    session,
-    formats: List[str],
-    params: ReportParams,
-) -> None:
+def report(args: Namespace, session, formats: List[str], params: ReportParams) -> None:
     """Write tabular report of pyani runs from database.
 
     :param args:  Namespace of command-line arguments

From 17f01fc56b0ff2d4ce322751d69f6f4cdf444495 Mon Sep 17 00:00:00 2001
From: baileythegreen <baileythegreen@gmail.com>
Date: Thu, 18 Nov 2021 22:04:06 +0000
Subject: [PATCH 05/33] Ensure genomes are sorted

---
 pyani/scripts/subcommands/subcmd_anim.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pyani/scripts/subcommands/subcmd_anim.py b/pyani/scripts/subcommands/subcmd_anim.py
index c71ea728..4037b61b 100644
--- a/pyani/scripts/subcommands/subcmd_anim.py
+++ b/pyani/scripts/subcommands/subcmd_anim.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # (c) The James Hutton Institute 2017-2019
-# (c) University of Strathclyde 2019-2020
+# (c) University of Strathclyde 2019-2022
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -18,7 +18,7 @@
 # The MIT License
 #
 # Copyright (c) 2017-2019 The James Hutton Institute
-# Copyright (c) 2019-2020 University of Strathclyde
+# Copyright (c) 2019-2022 University of Strathclyde
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -229,7 +229,7 @@ def subcmd_anim(args: Namespace) -> None:
 
     # Get list of genome IDs for this analysis from the database
     logger.info("Compiling genomes for comparison")
-    genomes = run.genomes.all()
+    genomes = sorted(run.genomes.all())
     logger.debug("Collected %s genomes for this run", len(genomes))
 
     # Generate all pair combinations of genome IDs as a list of (Genome, Genome) tuples

From 64964b6fe27564a27eb99804b5e655d89ee76a0c Mon Sep 17 00:00:00 2001
From: baileythegreen <baileythegreen@gmail.com>
Date: Fri, 19 Nov 2021 00:43:33 +0000
Subject: [PATCH 06/33] Fix genome sorting

---
 pyani/scripts/subcommands/subcmd_anim.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pyani/scripts/subcommands/subcmd_anim.py b/pyani/scripts/subcommands/subcmd_anim.py
index 4037b61b..be2c8670 100644
--- a/pyani/scripts/subcommands/subcmd_anim.py
+++ b/pyani/scripts/subcommands/subcmd_anim.py
@@ -229,7 +229,7 @@ def subcmd_anim(args: Namespace) -> None:
 
     # Get list of genome IDs for this analysis from the database
     logger.info("Compiling genomes for comparison")
-    genomes = sorted(run.genomes.all())
+    genomes = run.genomes.all()
     logger.debug("Collected %s genomes for this run", len(genomes))
 
     # Generate all pair combinations of genome IDs as a list of (Genome, Genome) tuples
@@ -320,10 +320,11 @@ def generate_joblist(
     existingfiles = set(existingfiles)  # Path objects hashable
 
     joblist = []  # will hold ComparisonJob structs
-    jobs = {"new": 0, "old": 0}  # will hold counts of new/old jobs for reporting
     for idx, (query, subject) in enumerate(
         tqdm(comparisons, disable=args.disable_tqdm)
     ):
+        if subject.path < query.path:
+            query, subject = subject, query  # sort them
         ncmd, dcmd = anim.construct_nucmer_cmdline(
             query.path,
             subject.path,

From 54610a69f2992a7d3d05b4411936dc994d2c3701 Mon Sep 17 00:00:00 2001
From: baileythegreen <baileythegreen@gmail.com>
Date: Mon, 13 Dec 2021 12:40:06 +0000
Subject: [PATCH 07/33] Change `nargs` for `pyani plot`

This should not ever have more than one value as files will just be
overwritten.
---
 pyani/scripts/parsers/plot_parser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyani/scripts/parsers/plot_parser.py b/pyani/scripts/parsers/plot_parser.py
index b35f8e01..e1729c6e 100644
--- a/pyani/scripts/parsers/plot_parser.py
+++ b/pyani/scripts/parsers/plot_parser.py
@@ -107,7 +107,7 @@ def build(
         action="store",
         default="seaborn",
         metavar="METHOD",
-        nargs=1,
+        nargs="?",
         choices=["seaborn", "mpl", "plotly"],
         help="graphics method to use for plotting; options (seaborn, mpl, plotly)",
     )

From 69c269b47d18d9d21e7ebef03e789aa03d884b38 Mon Sep 17 00:00:00 2001
From: baileythegreen <baileythegreen@gmail.com>
Date: Fri, 9 Jul 2021 19:26:29 +0100
Subject: [PATCH 08/33] Add `docs/subcmd_plot.rst`

---
 docs/subcmd_plot.rst | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/docs/subcmd_plot.rst b/docs/subcmd_plot.rst
index b0b7c595..84ee4cdb 100644
--- a/docs/subcmd_plot.rst
+++ b/docs/subcmd_plot.rst
@@ -8,13 +8,23 @@ The ``plot`` subcommand will plot the results of an ANI analysis, specified by `
 
 .. code-block:: text
 
-    usage: pyani plot [-h] [-l LOGFILE] [-v] [--debug] [--disable_tqdm] [--version]
-                  [--citation] -o OUTDIR --run_ids RUN_ID [RUN_ID ...]
-                  [--dbpath DBPATH] [--formats FORMAT [FORMAT ...]]
-                  [--method METHOD] [--workers WORKERS]
+    usage: pyani.py plot [-h] [-l LOGFILE] [-v] [--debug]
+    [--disable_tqdm] [--version] [--citation] [--dbpath DBPATH]
+    [--formats FORMATS] [--method {seaborn,mpl,plotly}]
+    outdir run_id
 
 .. _SQLite3: https://www.sqlite.org/index.html
 
+--------------------
+Positional arguments
+--------------------
+
+``outdir``
+   Path to a directory where comparison plot files will be written.
+
+``run_id``
+  Unique database ID of the run to be plotted.
+
 -----------------
 Flagged arguments
 -----------------
@@ -25,17 +35,11 @@ Flagged arguments
 ``--disable_tqdm``
     Disable the ``tqdm`` progress bar while the plotting process runs. This is useful when testing to avoid aesthetic problems with test output.
 
-``--formats FORMAT [FORMAT ...]``
+``--formats FORMATS``
     Graphics output format(s); more than one can be specified. Valid options are: (pdf/png/svg/jpg). (default: png)
 
 ``-l LOGFILE, --logfile LOGFILE``
     Provide the location ``LOGFILE`` to which a logfile of the plotting process will be written.
 
-``--method METHOD``
-    Graphics method to use for plotting; options (seaborn, mpl, plotly). (default: seaborn)
-
-``-o OUTDIR, --o outdir OUTDIR``
-    Path to a directory where comparison plot files will be written.
-
-``--run_ids RUN_ID [RUN_ID ...]``
-    Unique database ID of the runs to be plotted.
+``--method {seaborn,mpl,plotly}``
+    Graphics method to use for plotting. (default: seaborn)

From 606a908098efc8c70c8d592dda8460109a409e6a Mon Sep 17 00:00:00 2001
From: baileythegreen <baileythegreen@gmail.com>
Date: Fri, 9 Jul 2021 19:26:46 +0100
Subject: [PATCH 09/33] Add `docs/subcmd_report.rst`

I am less familiar with how this should work, and therefore this file
probably requires more scrutiny.
---
 docs/subcmd_report.rst | 36 +++++++++++++++++-------------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/docs/subcmd_report.rst b/docs/subcmd_report.rst
index 836d6e8a..c0750c9c 100644
--- a/docs/subcmd_report.rst
+++ b/docs/subcmd_report.rst
@@ -7,15 +7,20 @@
 The ``report`` subcommand reports the contents of a local `SQLite3`_ database, located at ``dbpath``. Output files will be written to the ``outdir`` directory.
 
 
-usage: pyani report [-h] [-l LOGFILE] [-v] [--debug] [--disable_tqdm] [--version]
-                    [--citation] -o OUTDIR [--dbpath DBPATH] [--runs] [--genomes]
-                    [--runs_genomes] [--genomes_runs]
-                    [--run_results RUN_ID [RUN_ID ...]]
-                    [--run_matrices RUN_ID [RUN_ID ...]] [--no_matrix_labels]
-                    [--formats FORMAT [FORMAT ...]]
+usage: pyani.py report [-h] [-l LOGFILE] [-v] [--debug] [--disable_tqdm] [--version]
+                       [--citation] [--dbpath DBPATH] [--runs] [--genomes]
+                       [--runs_genomes] [--genomes_runs] [--run_results RUN_RESULTS]
+                       [--run_matrices RUN_MATRICES] [--formats FORMATS]
+                       outdir
 
 .. _SQLite3: https://www.sqlite.org/index.html
 
+--------------------
+Positional arguments
+--------------------
+
+``outdir``
+    Directory where output analysis reports will be written.
 
 -----------------
 Flagged arguments
@@ -27,9 +32,8 @@ Flagged arguments
 ``--disable_tqdm``
     Disable the ``tqdm`` progress bar while the report process runs. This is useful when testing to avoid aesthetic problems with test output.
 
-``--formats FORMAT [FORMAT ...]``
-    Space-separated list of output formats (in addition to .tab);
-    possible values: {html, excel,stdout}. (default: None)
+``--formats FORMATS``
+    Output formats (in addition to .tab). (default: None)
 
 ``--genomes``
     Report table of genomes in database. (default: False)
@@ -37,23 +41,17 @@ Flagged arguments
 ``--genomes_runs``
     Report table of all runs in which each genome in the database participates. (default: False)
 
-``--no_matrix_labels``
-    Turn off row/column labels in output matrix files (default: False)
-
-``-o OUTDIR, --outdir OUTDIR``
-    Directory where output analysis reports will be written.
-
 ``--runs``
     Report table of analysis runs in database. (default: False)
 
 ``--runs_genomes``
     Report table of all genomes for each run in database. (default: False)
 
-``--run_matrices RUN_ID [RUN_ID ...]``
-    Report matrices of results for space-separated list of runs. (default: None)
+``--run_matrices RUN_MATRICES``
+    Report matrices of results for a pyani run. (default: False)
 
-``--run_results RUN_ID [RUN_ID ...]``
-    Report table of results for space-separated list of runs. (default: None)
+``--run_results RUN_RESULTS``
+    Report table of results for a pyani run. (default: False)
 
 ``-v, --verbose``
     Provide verbose output to ``STDOUT``.

From c55e3f2be9bd049995e6a55e4f08deb118001d2a Mon Sep 17 00:00:00 2001
From: baileythegreen <baileythegreen@gmail.com>
Date: Fri, 9 Jul 2021 19:37:44 +0100
Subject: [PATCH 10/33] Remove references to ANIm from ANIb documentation

---
 docs/run_anib.rst | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/run_anib.rst b/docs/run_anib.rst
index 59a81059..0c280dba 100644
--- a/docs/run_anib.rst
+++ b/docs/run_anib.rst
@@ -58,7 +58,7 @@ This command will write the intermediate `NCBI-BLAST+`_ output to the directory
   .. code-block:: bash
 
     $ ls genomes_ANIb/
-    blastn_output
+    nucmer_output
 
 .. ATTENTION::
     To view the output ANIb results, you will need to use the ``pyani report`` or ``pyani plot`` subcommands. Please see :ref:`pyani-subcmd-report` and :ref:`pyani-subcmd-plot` for more details.
@@ -67,11 +67,11 @@ This command will write the intermediate `NCBI-BLAST+`_ output to the directory
 Perform ANIb analysis with Open Grid Scheduler
 ----------------------------------------------
 
-The `NCBI-BLAST+`_ comparisons are embarrassingly parallel, and `NCBI-BLAST+`_ jobs can be distributed across cores in a cluster using the `Open Grid Scheduler`_. To enable this during the analysis, use the ``--scheduler SGE`` option:
+The `NCBI-BLAST+`_ comparisons are embarrasingly parallel, and these jobs can be distributed across cores in a cluster using the `Open Grid Scheduler`_. To enable this during the analysis, use the ``--scheduler SGE`` option:
 
 .. code-block:: bash
 
-    pyani anib --scheduler SGE -i genomes -o genomes_ANIb
+    pyani anib --scheduler SGE genomes genomes_ANIb
 
 .. NOTE::
     Jobs are submitted as *array jobs* to keep the scheduler queue short.
@@ -93,19 +93,19 @@ These allow for useful control of job execution. For example, the command:
 
 .. code-block:: bash
 
-    pyani anib --scheduler SGE --SGEgroupsize 5000 -i genomes -o genomes_ANIb
+    pyani anib --scheduler SGE --SGEgroupsize 5000 genomes genomes_ANIb
 
-will batch `NCBI-BLAST+`_ jobs in groups of 500 for the scheduler. The command:
+will batch ``ANIb`` jobs in groups of 500 for the scheduler. The command:
 
 .. code-block:: bash
 
-    pyani anib --scheduler SGE --jobprefix My_Ace_Job -i genomes -o genomes_ANIb
+    pyani anib --scheduler SGE --jobprefix My_Ace_Job genomes genomes_ANIb
 
 will prepend the string ``My_Ace_Job`` to your job in the scheduler queue. And the command:
 
 .. code-block:: bash
 
-    pyani anib --scheduler SGE --SGEargs "-m e -M my.name@my.domain" 5000 -i genomes -o genomes_ANIb
+    pyani anib --scheduler SGE --SGEargs "-m e -M my.name@my.domain" --SGEgroupsize 5000 genomes genomes_ANIb
 
 will email ``my.name@my.domain`` when the jobs finish.
 

From 4a3d7d54a9933fe170a2ba43c2e8bcb7122cffb1 Mon Sep 17 00:00:00 2001
From: baileythegreen <baileythegreen@gmail.com>
Date: Fri, 10 Sep 2021 17:19:57 +0100
Subject: [PATCH 11/33] Add test for NUCmer version parsing regex

---
 tests/conftest.py | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 8b876113..f480a910 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-# (c) The University of Strathclude 2019-2021
+# (c) The University of Strathclude 2019-2022
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -17,7 +17,7 @@
 #
 # The MIT License
 #
-# (c) The University of Strathclude 2019-2021
+# (c) The University of Strathclude 2019-2022
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -341,13 +341,6 @@ def mock_system(*args, **kwargs):
     monkeypatch.setattr(platform, "system", mock_system)
 
 
-@pytest.fixture
-def unsorted_genomes(dir_anim_in):
-    """Tests ordering of genome names in output file names for asymmetric analyses."""
-    dir_anim_in = str(dir_anim_in)
-    return (MockGenome(f"{dir_anim_in}/second"), MockGenome(f"{dir_anim_in}/first"))
-
-
 @pytest.fixture
 def mock_legacy_single_genome_dl(monkeypatch):
     """Mocks remote database calls for single-genome downloads.

From f9fb518ee5b84b7ede5cf977945f06716d5ecc1e Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 15:10:38 +0000
Subject: [PATCH 12/33] tidy test_mummer_single long filenames

---
 tests/test_anim.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/tests/test_anim.py b/tests/test_anim.py
index d3c40621..fca27465 100644
--- a/tests/test_anim.py
+++ b/tests/test_anim.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # (c) The James Hutton Institute 2017-2019
-# (c) University of Strathclyde 2019-2021
+# (c) University of Strathclyde 2019-2022
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -18,7 +18,7 @@
 # The MIT License
 #
 # Copyright (c) 2017-2019 The James Hutton Institute
-# Copyright (c) 2019-2021 University of Strathclyde
+# Copyright (c) 2019-2022 University of Strathclyde
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -251,16 +251,15 @@ def test_mummer_single(tmp_path, path_file_two):
         path_file_two[0], path_file_two[1], outdir=tmp_path
     )
     dir_nucmer = tmp_path / "nucmer_output"
+    outprefix = (
+        dir_nucmer
+        / str(path_file_two[0].stem)
+        / str(path_file_two[0].stem + "_vs_" + path_file_two[1].stem)
+    )
     expected = (
+        (f"nucmer --mum -p {outprefix} {path_file_two[0]} {path_file_two[1]}"),
         (
-            "nucmer --mum -p "
-            f"{dir_nucmer / str(path_file_two[0].stem) / str(path_file_two[0].stem + '_vs_' + path_file_two[1].stem)} "
-            f"{path_file_two[0]} {path_file_two[1]}"
-        ),
-        (
-            "delta_filter_wrapper.py delta-filter -1 "
-            f"{dir_nucmer / str(path_file_two[0].stem ) / str(path_file_two[0].stem + '_vs_' + path_file_two[1].stem + '.delta')} "
-            f"{dir_nucmer / str(path_file_two[0].stem ) / str(path_file_two[0].stem + '_vs_' + path_file_two[1].stem + '.filter')}"
+            f"delta_filter_wrapper.py delta-filter -1 {outprefix}.delta {outprefix}.filter"
         ),
     )
     assert cmds == expected

From c7d832712b685cc767795b4376d6bb164dd5a80d Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 15:12:13 +0000
Subject: [PATCH 13/33] update test_anim.py to use pandas.testing

---
 tests/test_anim.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_anim.py b/tests/test_anim.py
index fca27465..7bd8c75e 100644
--- a/tests/test_anim.py
+++ b/tests/test_anim.py
@@ -51,7 +51,7 @@
 import pytest
 import unittest
 
-from pandas.util.testing import assert_frame_equal
+from pandas.testing import assert_frame_equal
 
 from pyani import anim, pyani_files
 

From 70af0b305aa2edd12fa010fff1e47a7ac7c198e8 Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 15:16:07 +0000
Subject: [PATCH 14/33] update test_anim.py to use keywords for .sort_index()

---
 tests/test_anim.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_anim.py b/tests/test_anim.py
index 7bd8c75e..c4aa3866 100644
--- a/tests/test_anim.py
+++ b/tests/test_anim.py
@@ -208,8 +208,8 @@ def test_deltadir_parsing(delta_output_dir):
     orglengths = pyani_files.get_sequence_lengths(seqfiles)
     result = anim.process_deltadir(delta_output_dir.deltadir, orglengths)
     assert_frame_equal(
-        result.percentage_identity.sort_index(1).sort_index(),
-        delta_output_dir.deltaresult.sort_index(1).sort_index(),
+        result.percentage_identity.sort_index(axis=1).sort_index(),
+        delta_output_dir.deltaresult.sort_index(axis=1).sort_index(),
     )
 
 

From 03be4b06f57f55e167e3107a892bd94d0af2a610 Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 15:21:26 +0000
Subject: [PATCH 15/33] remove unused import, unnecessary parentheses

---
 tests/test_anim.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/tests/test_anim.py b/tests/test_anim.py
index c4aa3866..baf5b5b9 100644
--- a/tests/test_anim.py
+++ b/tests/test_anim.py
@@ -43,7 +43,7 @@
 
 pytest -v
 """
-import sys
+
 from pathlib import Path
 from typing import List, NamedTuple, Tuple
 
@@ -257,10 +257,8 @@ def test_mummer_single(tmp_path, path_file_two):
         / str(path_file_two[0].stem + "_vs_" + path_file_two[1].stem)
     )
     expected = (
-        (f"nucmer --mum -p {outprefix} {path_file_two[0]} {path_file_two[1]}"),
-        (
-            f"delta_filter_wrapper.py delta-filter -1 {outprefix}.delta {outprefix}.filter"
-        ),
+        f"nucmer --mum -p {outprefix} {path_file_two[0]} {path_file_two[1]}",
+        f"delta_filter_wrapper.py delta-filter -1 {outprefix}.delta {outprefix}.filter",
     )
     assert cmds == expected
 

From c0d09ec2ca8e10bed1cd3847db5f1ebc90af9513 Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 15:53:09 +0000
Subject: [PATCH 16/33] restore changes to ANIb docs

---
 docs/run_anib.rst | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/run_anib.rst b/docs/run_anib.rst
index 0c280dba..59a81059 100644
--- a/docs/run_anib.rst
+++ b/docs/run_anib.rst
@@ -58,7 +58,7 @@ This command will write the intermediate `NCBI-BLAST+`_ output to the directory
   .. code-block:: bash
 
     $ ls genomes_ANIb/
-    nucmer_output
+    blastn_output
 
 .. ATTENTION::
     To view the output ANIb results, you will need to use the ``pyani report`` or ``pyani plot`` subcommands. Please see :ref:`pyani-subcmd-report` and :ref:`pyani-subcmd-plot` for more details.
@@ -67,11 +67,11 @@ This command will write the intermediate `NCBI-BLAST+`_ output to the directory
 Perform ANIb analysis with Open Grid Scheduler
 ----------------------------------------------
 
-The `NCBI-BLAST+`_ comparisons are embarrasingly parallel, and these jobs can be distributed across cores in a cluster using the `Open Grid Scheduler`_. To enable this during the analysis, use the ``--scheduler SGE`` option:
+The `NCBI-BLAST+`_ comparisons are embarrassingly parallel, and `NCBI-BLAST+`_ jobs can be distributed across cores in a cluster using the `Open Grid Scheduler`_. To enable this during the analysis, use the ``--scheduler SGE`` option:
 
 .. code-block:: bash
 
-    pyani anib --scheduler SGE genomes genomes_ANIb
+    pyani anib --scheduler SGE -i genomes -o genomes_ANIb
 
 .. NOTE::
     Jobs are submitted as *array jobs* to keep the scheduler queue short.
@@ -93,19 +93,19 @@ These allow for useful control of job execution. For example, the command:
 
 .. code-block:: bash
 
-    pyani anib --scheduler SGE --SGEgroupsize 5000 genomes genomes_ANIb
+    pyani anib --scheduler SGE --SGEgroupsize 5000 -i genomes -o genomes_ANIb
 
-will batch ``ANIb`` jobs in groups of 500 for the scheduler. The command:
+will batch `NCBI-BLAST+`_ jobs in groups of 500 for the scheduler. The command:
 
 .. code-block:: bash
 
-    pyani anib --scheduler SGE --jobprefix My_Ace_Job genomes genomes_ANIb
+    pyani anib --scheduler SGE --jobprefix My_Ace_Job -i genomes -o genomes_ANIb
 
 will prepend the string ``My_Ace_Job`` to your job in the scheduler queue. And the command:
 
 .. code-block:: bash
 
-    pyani anib --scheduler SGE --SGEargs "-m e -M my.name@my.domain" --SGEgroupsize 5000 genomes genomes_ANIb
+    pyani anib --scheduler SGE --SGEargs "-m e -M my.name@my.domain" 5000 -i genomes -o genomes_ANIb
 
 will email ``my.name@my.domain`` when the jobs finish.
 

From 0cdf785e0ddd72c95e17066f52625739d55b1e57 Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 15:54:37 +0000
Subject: [PATCH 17/33] restored changes to docs for subcmd_plot

---
 docs/subcmd_plot.rst | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/docs/subcmd_plot.rst b/docs/subcmd_plot.rst
index 84ee4cdb..ce18eb87 100644
--- a/docs/subcmd_plot.rst
+++ b/docs/subcmd_plot.rst
@@ -8,10 +8,10 @@ The ``plot`` subcommand will plot the results of an ANI analysis, specified by `
 
 .. code-block:: text
 
-    usage: pyani.py plot [-h] [-l LOGFILE] [-v] [--debug]
-    [--disable_tqdm] [--version] [--citation] [--dbpath DBPATH]
-    [--formats FORMATS] [--method {seaborn,mpl,plotly}]
-    outdir run_id
+    usage: pyani plot [-h] [-l LOGFILE] [-v] [--debug] [--disable_tqdm] [--version]
+                  [--citation] -o OUTDIR --run_ids RUN_ID [RUN_ID ...]
+                  [--dbpath DBPATH] [--formats FORMAT [FORMAT ...]]
+                  [--method METHOD] [--workers WORKERS]
 
 .. _SQLite3: https://www.sqlite.org/index.html
 
@@ -35,11 +35,17 @@ Flagged arguments
 ``--disable_tqdm``
     Disable the ``tqdm`` progress bar while the plotting process runs. This is useful when testing to avoid aesthetic problems with test output.
 
-``--formats FORMATS``
+``--formats FORMAT [FORMAT ...]``
     Graphics output format(s); more than one can be specified. Valid options are: (pdf/png/svg/jpg). (default: png)
 
 ``-l LOGFILE, --logfile LOGFILE``
     Provide the location ``LOGFILE`` to which a logfile of the plotting process will be written.
 
-``--method {seaborn,mpl,plotly}``
-    Graphics method to use for plotting. (default: seaborn)
+``--method METHOD``
+    Graphics method to use for plotting; options (seaborn, mpl, plotly). (default: seaborn)
+
+``-o OUTDIR, --o outdir OUTDIR``
+    Path to a directory where comparison plot files will be written.
+
+``--run_ids RUN_ID [RUN_ID ...]``
+    Unique database ID of the runs to be plotted.

From 1f42ddad37714b32a2b370c3ac6124cd536f0578 Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 15:55:04 +0000
Subject: [PATCH 18/33] restore changes to docs for subcmd_report

---
 docs/subcmd_report.rst | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/docs/subcmd_report.rst b/docs/subcmd_report.rst
index c0750c9c..d396f6bd 100644
--- a/docs/subcmd_report.rst
+++ b/docs/subcmd_report.rst
@@ -7,11 +7,12 @@
 The ``report`` subcommand reports the contents of a local `SQLite3`_ database, located at ``dbpath``. Output files will be written to the ``outdir`` directory.
 
 
-usage: pyani.py report [-h] [-l LOGFILE] [-v] [--debug] [--disable_tqdm] [--version]
-                       [--citation] [--dbpath DBPATH] [--runs] [--genomes]
-                       [--runs_genomes] [--genomes_runs] [--run_results RUN_RESULTS]
-                       [--run_matrices RUN_MATRICES] [--formats FORMATS]
-                       outdir
+usage: pyani report [-h] [-l LOGFILE] [-v] [--debug] [--disable_tqdm] [--version]
+                    [--citation] -o OUTDIR [--dbpath DBPATH] [--runs] [--genomes]
+                    [--runs_genomes] [--genomes_runs]
+                    [--run_results RUN_ID [RUN_ID ...]]
+                    [--run_matrices RUN_ID [RUN_ID ...]] [--no_matrix_labels]
+                    [--formats FORMAT [FORMAT ...]]
 
 .. _SQLite3: https://www.sqlite.org/index.html
 
@@ -32,8 +33,9 @@ Flagged arguments
 ``--disable_tqdm``
     Disable the ``tqdm`` progress bar while the report process runs. This is useful when testing to avoid aesthetic problems with test output.
 
-``--formats FORMATS``
-    Output formats (in addition to .tab). (default: None)
+``--formats FORMAT [FORMAT ...]``
+    Space-separated list of output formats (in addition to .tab);
+    possible values: {html, excel,stdout}. (default: None)
 
 ``--genomes``
     Report table of genomes in database. (default: False)
@@ -41,17 +43,23 @@ Flagged arguments
 ``--genomes_runs``
     Report table of all runs in which each genome in the database participates. (default: False)
 
+``--no_matrix_labels``
+    Turn off row/column labels in output matrix files (default: False)
+
+``-o OUTDIR, --outdir OUTDIR``
+    Directory where output analysis reports will be written.
+
 ``--runs``
     Report table of analysis runs in database. (default: False)
 
 ``--runs_genomes``
     Report table of all genomes for each run in database. (default: False)
 
-``--run_matrices RUN_MATRICES``
-    Report matrices of results for a pyani run. (default: False)
+``--run_matrices RUN_ID [RUN_ID ...]``
+    Report matrices of results for space-separated list of runs. (default: None)
 
-``--run_results RUN_RESULTS``
-    Report table of results for a pyani run. (default: False)
+``--run_results RUN_ID [RUN_ID ...]``
+    Report table of results for space-separated list of runs. (default: None)
 
 ``-v, --verbose``
     Provide verbose output to ``STDOUT``.

From 98f47ba87c4abd57c9540230a4f36efb5c4f59f2 Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 15:56:17 +0000
Subject: [PATCH 19/33] restore changes to subcmd_plot.py

---
 pyani/scripts/subcommands/subcmd_plot.py | 33 +++++++++++++++++-------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/pyani/scripts/subcommands/subcmd_plot.py b/pyani/scripts/subcommands/subcmd_plot.py
index 8070ed42..6cd09915 100644
--- a/pyani/scripts/subcommands/subcmd_plot.py
+++ b/pyani/scripts/subcommands/subcmd_plot.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # (c) The James Hutton Institute 2017-2019
-# (c) University of Strathclyde 2019-2022
+# (c) University of Strathclyde 2019-2020
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -18,7 +18,7 @@
 # The MIT License
 #
 # Copyright (c) 2017-2019 The James Hutton Institute
-# Copyright (c) 2019-2022 University of Strathclyde
+# Copyright (c) 2019-2020 University of Strathclyde
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -117,11 +117,6 @@ def write_run_plots(run_id: int, session, outfmts: List[str], args: Namespace) -
     logger.debug(
         f"Have {len(result_label_dict)} labels and {len(result_class_dict)} classes"
     )
-    result_label_dict = pyani_orm.get_matrix_labels_for_run(session, args.run_id)
-    result_class_dict = pyani_orm.get_matrix_classes_for_run(session, args.run_id)
-    logger.debug(
-        f"Have {len(result_label_dict)} labels and {len(result_class_dict)} classes"
-    )
 
     # Write heatmap and distribution plot for each results matrix
 
@@ -146,9 +141,24 @@ def write_run_plots(run_id: int, session, outfmts: List[str], args: Namespace) -
                 [run_id, matdata, result_label_dict, result_class_dict, outfmts, args],
             )
         )
-        plotting_commands.append(
-            (write_distribution, [run_id, matdata, outfmts, args])  # type: ignore
+        plotting_commands.append((write_distribution, [run_id, matdata, outfmts, args]))
+
+    id_matrix = MatrixData("identity", pd.read_json(results.df_identity), {})
+    cov_matrix = MatrixData("coverage", pd.read_json(results.df_coverage), {})
+    plotting_commands.append(
+        (
+            write_scatter,
+            [
+                run_id,
+                id_matrix,
+                cov_matrix,
+                result_label_dict,
+                result_class_dict,
+                outfmts,
+                args,
+            ],
         )
+    )
 
     # Run the plotting commands
     [pool.apply_async(func, args, {}) for func, args in plotting_commands]
@@ -169,7 +179,10 @@ def write_run_plots(run_id: int, session, outfmts: List[str], args: Namespace) -
 
 
 def write_distribution(
-    run_id: int, matdata: MatrixData, outfmts: List[str], args: Namespace
+    run_id: int,
+    matdata: MatrixData,
+    outfmts: List[str],
+    args: Namespace,
 ) -> None:
     """Write distribution plots for each matrix type.
 

From e4c620c25261b36be6381771f2552e60ff237f5e Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 15:58:36 +0000
Subject: [PATCH 20/33] restore changes to subcmd_report.py

---
 pyani/scripts/subcommands/subcmd_report.py | 27 +++++++++++++++++-----
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/pyani/scripts/subcommands/subcmd_report.py b/pyani/scripts/subcommands/subcmd_report.py
index 81d65f88..82423893 100644
--- a/pyani/scripts/subcommands/subcmd_report.py
+++ b/pyani/scripts/subcommands/subcmd_report.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # (c) The James Hutton Institute 2017-2019
-# (c) University of Strathclyde 2019-2022
+# (c) University of Strathclyde 2019-2020
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -18,7 +18,7 @@
 # The MIT License
 #
 # Copyright (c) 2017-2019 The James Hutton Institute
-# Copyright (c) 2019-2022 University of Strathclyde
+# Copyright (c) 2019-2020 University of Strathclyde
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -154,7 +154,12 @@ def subcmd_report(args: Namespace) -> int:
             "genome label",
             "genome class",
         ]
-        report(args, session, formats, ReportParams("runs_genomes", statement, headers))
+        report(
+            args,
+            session,
+            formats,
+            ReportParams("runs_genomes", statement, headers),
+        )
 
     # Report table of all runs in which a genome is involved
     if args.show_genomes_runs:
@@ -191,7 +196,12 @@ def subcmd_report(args: Namespace) -> int:
             "method",
             "date run",
         ]
-        report(args, session, formats, ReportParams("genomes_runs", statement, headers))
+        report(
+            args,
+            session,
+            formats,
+            ReportParams("genomes_runs", statement, headers),
+        )
 
     # Report table of comparison results for the indicated runs
     if args.run_results:
@@ -256,7 +266,7 @@ def subcmd_report(args: Namespace) -> int:
     # our matrices directly, here
     if args.run_matrices:
         show_index = not args.no_matrix_labels
-        for run_id in [run_id.strip() for run_id in args.run_matrices.split(",")]:
+        for run_id in [run_id for run_id in args.run_matrices]:
             logger.debug("Extracting matrices for run %s", run_id)
             run = session.query(Run).filter(Run.run_id == run_id).first()
             matlabel_dict = get_matrix_labels_for_run(session, run_id)
@@ -290,7 +300,12 @@ def subcmd_report(args: Namespace) -> int:
     return 0
 
 
-def report(args: Namespace, session, formats: List[str], params: ReportParams) -> None:
+def report(
+    args: Namespace,
+    session,
+    formats: List[str],
+    params: ReportParams,
+) -> None:
     """Write tabular report of pyani runs from database.
 
     :param args:  Namespace of command-line arguments

From eed2872bf8c01e47a0db47f27e0940e11375b5e0 Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 15:58:58 +0000
Subject: [PATCH 21/33] restore changes to plot_parser.py and subcmd_anim.py

---
 pyani/scripts/parsers/plot_parser.py     | 2 +-
 pyani/scripts/subcommands/subcmd_anim.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/pyani/scripts/parsers/plot_parser.py b/pyani/scripts/parsers/plot_parser.py
index e1729c6e..b35f8e01 100644
--- a/pyani/scripts/parsers/plot_parser.py
+++ b/pyani/scripts/parsers/plot_parser.py
@@ -107,7 +107,7 @@ def build(
         action="store",
         default="seaborn",
         metavar="METHOD",
-        nargs="?",
+        nargs=1,
         choices=["seaborn", "mpl", "plotly"],
         help="graphics method to use for plotting; options (seaborn, mpl, plotly)",
     )
diff --git a/pyani/scripts/subcommands/subcmd_anim.py b/pyani/scripts/subcommands/subcmd_anim.py
index be2c8670..d0e68df4 100644
--- a/pyani/scripts/subcommands/subcmd_anim.py
+++ b/pyani/scripts/subcommands/subcmd_anim.py
@@ -320,6 +320,7 @@ def generate_joblist(
     existingfiles = set(existingfiles)  # Path objects hashable
 
     joblist = []  # will hold ComparisonJob structs
+    jobs = {"new": 0, "old": 0}  # will hold counts of new/old jobs for reporting
     for idx, (query, subject) in enumerate(
         tqdm(comparisons, disable=args.disable_tqdm)
     ):

From 3ee671ed4a8b5005c617894c086fe08be2496d55 Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 16:02:37 +0000
Subject: [PATCH 22/33] allow exception to report an identifier rather than an
 error

---
 pyani/scripts/subcommands/subcmd_anim.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyani/scripts/subcommands/subcmd_anim.py b/pyani/scripts/subcommands/subcmd_anim.py
index d0e68df4..2c5af8c3 100644
--- a/pyani/scripts/subcommands/subcmd_anim.py
+++ b/pyani/scripts/subcommands/subcmd_anim.py
@@ -196,10 +196,10 @@ def subcmd_anim(args: Namespace) -> None:
         )
     except PyaniORMException:
         logger.error(
-            "Could not add run %s to the database (exiting)", run, exc_info=True
+            "Could not add run %s to the database (exiting)", name, exc_info=True
         )
         raise SystemExit(1)
-    logger.debug("...added run ID: %s to the database", run)
+    logger.debug("...added run ID: %s (%s) to the database", run, name)
 
     # Identify input files for comparison, and populate the database
     logger.debug("Adding genomes for run %s to database...", run)

From cf2fe0eea3201ed48a3567046096aa8a32b115f6 Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 16:35:40 +0000
Subject: [PATCH 23/33] update Makefile for new report CLI

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 90678b94..629ff1f0 100644
--- a/Makefile
+++ b/Makefile
@@ -47,7 +47,7 @@ walkthrough: clean_walkthrough
 	pyani report --runs -o C_blochmannia_ANIm/ --formats html excel stdout
 	pyani report --run_results 1 --formats html excel stdout -o C_blochmannia_ANIm/
 	pyani report --run_matrices 1 --formats html excel stdout -o C_blochmannia_ANIm/
-	pyani plot --formats png pdf --method seaborn -o C_blochmannia_ANIm --run_ids 1
+	pyani plot --formats png pdf --method seaborn -o C_blochmannia_ANIm --run_id 1
 	# pyani anib C_blochmannia C_blochmannia_ANIb \
     #     --name "C. blochmannia run 2" \
     #     --labels C_blochmannia/labels.txt --classes C_blochmannia/classes.txt

From 9c4704bf2660aae35c83a87976b5f58641bdb9ed Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 18:18:01 +0000
Subject: [PATCH 24/33] bring plotting up to date with master

---
 docs/subcmd_plot.rst                     | 10 ----------
 docs/subcmd_report.rst                   |  6 ------
 pyani/scripts/subcommands/subcmd_plot.py | 16 +++-------------
 3 files changed, 3 insertions(+), 29 deletions(-)

diff --git a/docs/subcmd_plot.rst b/docs/subcmd_plot.rst
index ce18eb87..b0b7c595 100644
--- a/docs/subcmd_plot.rst
+++ b/docs/subcmd_plot.rst
@@ -15,16 +15,6 @@ The ``plot`` subcommand will plot the results of an ANI analysis, specified by `
 
 .. _SQLite3: https://www.sqlite.org/index.html
 
---------------------
-Positional arguments
---------------------
-
-``outdir``
-   Path to a directory where comparison plot files will be written.
-
-``run_id``
-  Unique database ID of the run to be plotted.
-
 -----------------
 Flagged arguments
 -----------------
diff --git a/docs/subcmd_report.rst b/docs/subcmd_report.rst
index d396f6bd..836d6e8a 100644
--- a/docs/subcmd_report.rst
+++ b/docs/subcmd_report.rst
@@ -16,12 +16,6 @@ usage: pyani report [-h] [-l LOGFILE] [-v] [--debug] [--disable_tqdm] [--version
 
 .. _SQLite3: https://www.sqlite.org/index.html
 
---------------------
-Positional arguments
---------------------
-
-``outdir``
-    Directory where output analysis reports will be written.
 
 -----------------
 Flagged arguments
diff --git a/pyani/scripts/subcommands/subcmd_plot.py b/pyani/scripts/subcommands/subcmd_plot.py
index 6cd09915..d3682d1d 100644
--- a/pyani/scripts/subcommands/subcmd_plot.py
+++ b/pyani/scripts/subcommands/subcmd_plot.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # (c) The James Hutton Institute 2017-2019
-# (c) University of Strathclyde 2019-2020
+# (c) University of Strathclyde 2019-2022
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -18,7 +18,7 @@
 # The MIT License
 #
 # Copyright (c) 2017-2019 The James Hutton Institute
-# Copyright (c) 2019-2020 University of Strathclyde
+# Copyright (c) 2019-2022 University of Strathclyde
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -160,13 +160,6 @@ def write_run_plots(run_id: int, session, outfmts: List[str], args: Namespace) -
         )
     )
 
-    # Run the plotting commands
-    [pool.apply_async(func, args, {}) for func, args in plotting_commands]
-
-    # Close worker pool
-    pool.close()
-    pool.join()
-
     # Run the plotting commands
     logger.debug("Running plotting commands")
     for func, options in plotting_commands:
@@ -179,10 +172,7 @@ def write_run_plots(run_id: int, session, outfmts: List[str], args: Namespace) -
 
 
 def write_distribution(
-    run_id: int,
-    matdata: MatrixData,
-    outfmts: List[str],
-    args: Namespace,
+    run_id: int, matdata: MatrixData, outfmts: List[str], args: Namespace
 ) -> None:
     """Write distribution plots for each matrix type.
 

From f52be296003a19f5bd79e66be7d9dfe9db39e246 Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 18:19:13 +0000
Subject: [PATCH 25/33] bring ANIm up to date with master

---
 pyani/pyani_tools.py                     | 13 +++++++++----
 pyani/scripts/subcommands/subcmd_anim.py | 10 ++++------
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/pyani/pyani_tools.py b/pyani/pyani_tools.py
index ae5ab8ad..a9b93ae8 100644
--- a/pyani/pyani_tools.py
+++ b/pyani/pyani_tools.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # (c) The James Hutton Institute 2013-2019
-# (c) The University of Strathclude 2019-2022
+# (c) The University of Strathclude 2019-2020
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -19,7 +19,7 @@
 # The MIT License
 #
 # Copyright (c) 2013-2019 The James Hutton Institute
-# (c) The University of Strathclude 2019-2022
+# (c) The University of Strathclude 2019-2020
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -303,8 +303,13 @@ def label_results_matrix(matrix: pd.DataFrame, labels: Dict) -> pd.DataFrame:
     Applies the labels from the dictionary to the dataframe in
     matrix, and returns the result.
     """
-    matrix.columns = [f"{labels.get(_, _)}:{_}" for _ in matrix.columns]
-    matrix.index = [f"{labels.get(_, _)}:{_}" for _ in matrix.index]
+    # The dictionary uses string keys!
+    # Create a label function that produces <label>:<genome_id>
+    # when a label is available; and just Genome_id:<genome_id> when no
+    # label exists
+    label = lambda gen_id: f"{labels.get(str(gen_id), 'Genome_id')}:{gen_id}"
+    matrix.columns = [label(_) for _ in matrix.columns]
+    matrix.index = [label(_) for _ in matrix.index]
     return matrix
 
 
diff --git a/pyani/scripts/subcommands/subcmd_anim.py b/pyani/scripts/subcommands/subcmd_anim.py
index 2c5af8c3..c71ea728 100644
--- a/pyani/scripts/subcommands/subcmd_anim.py
+++ b/pyani/scripts/subcommands/subcmd_anim.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # (c) The James Hutton Institute 2017-2019
-# (c) University of Strathclyde 2019-2022
+# (c) University of Strathclyde 2019-2020
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -18,7 +18,7 @@
 # The MIT License
 #
 # Copyright (c) 2017-2019 The James Hutton Institute
-# Copyright (c) 2019-2022 University of Strathclyde
+# Copyright (c) 2019-2020 University of Strathclyde
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -196,10 +196,10 @@ def subcmd_anim(args: Namespace) -> None:
         )
     except PyaniORMException:
         logger.error(
-            "Could not add run %s to the database (exiting)", name, exc_info=True
+            "Could not add run %s to the database (exiting)", run, exc_info=True
         )
         raise SystemExit(1)
-    logger.debug("...added run ID: %s (%s) to the database", run, name)
+    logger.debug("...added run ID: %s to the database", run)
 
     # Identify input files for comparison, and populate the database
     logger.debug("Adding genomes for run %s to database...", run)
@@ -324,8 +324,6 @@ def generate_joblist(
     for idx, (query, subject) in enumerate(
         tqdm(comparisons, disable=args.disable_tqdm)
     ):
-        if subject.path < query.path:
-            query, subject = subject, query  # sort them
         ncmd, dcmd = anim.construct_nucmer_cmdline(
             query.path,
             subject.path,

From 1e14db6188d0bfd8b298628fc42c6fde4ef499e7 Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Wed, 19 Jan 2022 18:20:16 +0000
Subject: [PATCH 26/33] bring tests up to date with master

---
 tests/conftest.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index f480a910..8b876113 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-# (c) The University of Strathclude 2019-2022
+# (c) The University of Strathclude 2019-2021
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -17,7 +17,7 @@
 #
 # The MIT License
 #
-# (c) The University of Strathclude 2019-2022
+# (c) The University of Strathclude 2019-2021
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -341,6 +341,13 @@ def mock_system(*args, **kwargs):
     monkeypatch.setattr(platform, "system", mock_system)
 
 
+@pytest.fixture
+def unsorted_genomes(dir_anim_in):
+    """Tests ordering of genome names in output file names for asymmetric analyses."""
+    dir_anim_in = str(dir_anim_in)
+    return (MockGenome(f"{dir_anim_in}/second"), MockGenome(f"{dir_anim_in}/first"))
+
+
 @pytest.fixture
 def mock_legacy_single_genome_dl(monkeypatch):
     """Mocks remote database calls for single-genome downloads.

From fdb57438d34edaedc85c860725f1306d22ba17cb Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Thu, 20 Jan 2022 13:13:55 +0000
Subject: [PATCH 27/33] fix to resolve pandas deprecation warning

---
 tests/test_tetra.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_tetra.py b/tests/test_tetra.py
index 752dcd47..b1e13722 100644
--- a/tests/test_tetra.py
+++ b/tests/test_tetra.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # (c) The James Hutton Institute 2017-2019
-# (c) The University of Strathclude 2019-2020
+# (c) The University of Strathclude 2019-2022
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -19,7 +19,7 @@
 # The MIT License
 #
 # Copyright (c) 2017-2019 The James Hutton Institute
-# (c) The University of Strathclude 2019-2020
+# (c) The University of Strathclude 2019-2022
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -52,7 +52,7 @@
 
 import pandas as pd
 
-from pandas.util.testing import assert_frame_equal
+from pandas.testing import assert_frame_equal
 
 from pyani.tetra import (
     calculate_correlations,

From 6013541d8740fd6f9eaa8b44e809f26ea096d1e0 Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Thu, 20 Jan 2022 13:15:39 +0000
Subject: [PATCH 28/33] fix to avoid pandas future deprecation, and remove
 unused imports

---
 tests/test_anib.py  | 13 ++++++-------
 tests/test_tetra.py |  3 ---
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/tests/test_anib.py b/tests/test_anib.py
index 2f8d2380..f522a107 100644
--- a/tests/test_anib.py
+++ b/tests/test_anib.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # (c) The James Hutton Institute 2017-2019
-# (c) University of Strathclyde 2019-2020
+# (c) University of Strathclyde 2019-2022
 # Author: Leighton Pritchard
 #
 # Contact: leighton.pritchard@strath.ac.uk
@@ -17,7 +17,7 @@
 # The MIT License
 #
 # Copyright (c) 2017-2019 The James Hutton Institute
-# Copyright (c) 2019-2020 University of Strathclyde
+# Copyright (c) 2019-2022 University of Strathclyde
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -49,7 +49,6 @@
 import pandas as pd
 import pytest  # noqa: F401  # pylint: disable=unused-import
 import unittest
-import unittest.mock as mock
 
 from pandas.testing import assert_frame_equal
 
@@ -367,8 +366,8 @@ def test_parse_legacy_blastdir(anib_output_dir):
         anib_output_dir.legacyblastdir, orglengths, fraglengths, mode="ANIblastall"
     )
     assert_frame_equal(
-        result.percentage_identity.sort_index(1).sort_index(),
-        anib_output_dir.legacyblastresult.sort_index(1).sort_index(),
+        result.percentage_identity.sort_index(axis=1).sort_index(),
+        anib_output_dir.legacyblastresult.sort_index(axis=1).sort_index(),
     )
 
 
@@ -380,8 +379,8 @@ def test_parse_blastdir(anib_output_dir):
         anib_output_dir.blastdir, orglengths, fraglengths, mode="ANIb"
     )
     assert_frame_equal(
-        result.percentage_identity.sort_index(1).sort_index(),
-        anib_output_dir.blastresult.sort_index(1).sort_index(),
+        result.percentage_identity.sort_index(axis=1).sort_index(),
+        anib_output_dir.blastresult.sort_index(axis=1).sort_index(),
     )
 
 
diff --git a/tests/test_tetra.py b/tests/test_tetra.py
index b1e13722..13507c9d 100644
--- a/tests/test_tetra.py
+++ b/tests/test_tetra.py
@@ -46,9 +46,6 @@
 """
 
 import json
-import unittest
-
-from pathlib import Path
 
 import pandas as pd
 

From 735fe7fc29b01ad2f897ff648a945be3cf4eef49 Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Thu, 20 Jan 2022 13:54:46 +0000
Subject: [PATCH 29/33] suppress non-relevant scipy cluster warning about
 distance matrices

---
 pyani/pyani_graphics/sns/__init__.py         | 36 ++++++++++++++------
 pyani/scripts/average_nucleotide_identity.py |  1 -
 2 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/pyani/pyani_graphics/sns/__init__.py b/pyani/pyani_graphics/sns/__init__.py
index c8bcb284..8d4fe73b 100644
--- a/pyani/pyani_graphics/sns/__init__.py
+++ b/pyani/pyani_graphics/sns/__init__.py
@@ -37,6 +37,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 """Code to implement Seaborn graphics output for ANI analyses."""
+import warnings
 
 import matplotlib  # pylint: disable=C0411
 import pandas as pd
@@ -106,17 +107,30 @@ def get_clustermap(dfr, params, title=None, annot=True):
     :param title:  str, plot title
     :param annot:  Boolean, add text for cell values?
     """
-    fig = sns.clustermap(
-        dfr,
-        cmap=params.cmap,
-        vmin=params.vmin,
-        vmax=params.vmax,
-        col_colors=params.colorbar,
-        row_colors=params.colorbar,
-        figsize=(params.figsize, params.figsize),
-        linewidths=params.linewidths,
-        annot=annot,
-    )
+
+    # If we do not catch warnings here, then we often get the following warning:
+    #   ClusterWarning: scipy.cluster: The symmetric non-negative hollow
+    #   observation matrix looks suspiciously like an uncondensed distance matrix
+    # The usual solution would be to convert the array with
+    # scipy.spatial.distance.squareform(), but this requires that all values in
+    # the main diagonal are zero, which is not the case for ANI.
+    # As we know this is a (1-distance) matrix, we could just set the diagonal
+    # to zero and fudge it, but this is not a good solution. Instead, we suppress
+    # the warning in a context manager for this function call only, because we
+    # know the warning is not relevant.
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        fig = sns.clustermap(
+            dfr,
+            cmap=params.cmap,
+            vmin=params.vmin,
+            vmax=params.vmax,
+            col_colors=params.colorbar,
+            row_colors=params.colorbar,
+            figsize=(params.figsize, params.figsize),
+            linewidths=params.linewidths,
+            annot=annot,
+        )
 
     # add labels for each of the input genomes
     add_labels(fig, params)
diff --git a/pyani/scripts/average_nucleotide_identity.py b/pyani/scripts/average_nucleotide_identity.py
index 10fc7767..0fc0dcbe 100755
--- a/pyani/scripts/average_nucleotide_identity.py
+++ b/pyani/scripts/average_nucleotide_identity.py
@@ -812,7 +812,6 @@ def draw(args: Namespace, filestems: List[str], gformat: str) -> None:
         infilename = fullstem.with_suffix(".tab")
         dfm = pd.read_csv(infilename, index_col=0, sep="\t")
         logger.info("Writing heatmap to %s", outfilename)
-        print(args.labels, args.classes)
         params = pyani_graphics.Params(
             params_mpl(dfm)[filestem],
             pyani_tools.get_labels(args.labels, logger=logger),

From 25b4a37b2f35851340b1879afc039c73196454a2 Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Thu, 20 Jan 2022 15:34:10 +0000
Subject: [PATCH 30/33] make warning filter more precise

---
 pyani/pyani_graphics/sns/__init__.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/pyani/pyani_graphics/sns/__init__.py b/pyani/pyani_graphics/sns/__init__.py
index 8d4fe73b..fdcddea5 100644
--- a/pyani/pyani_graphics/sns/__init__.py
+++ b/pyani/pyani_graphics/sns/__init__.py
@@ -119,7 +119,14 @@ def get_clustermap(dfr, params, title=None, annot=True):
     # the warning in a context manager for this function call only, because we
     # know the warning is not relevant.
     with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
+        warnings.filterwarnings(
+            "ignore",
+            message=(
+                "scipy.cluster: The symmetric non-negative "
+                "hollow observation matrix looks suspiciously like an "
+                "uncondensed distance matrix"
+            ),
+        )
         fig = sns.clustermap(
             dfr,
             cmap=params.cmap,

From 266d823807781cb30ca565b7c94ae20d4875fb3b Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Thu, 20 Jan 2022 15:57:20 +0000
Subject: [PATCH 31/33] suppress warning about SQL JOIN statements arising from
 pandas

---
 pyani/scripts/subcommands/subcmd_report.py | 47 ++++++++++++----------
 tests/test_subcmd_05_report.py             |  4 +-
 2 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/pyani/scripts/subcommands/subcmd_report.py b/pyani/scripts/subcommands/subcmd_report.py
index 82423893..c9d74a78 100644
--- a/pyani/scripts/subcommands/subcmd_report.py
+++ b/pyani/scripts/subcommands/subcmd_report.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # (c) The James Hutton Institute 2017-2019
-# (c) University of Strathclyde 2019-2020
+# (c) University of Strathclyde 2019-2022
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -18,7 +18,7 @@
 # The MIT License
 #
 # Copyright (c) 2017-2019 The James Hutton Institute
-# Copyright (c) 2019-2020 University of Strathclyde
+# Copyright (c) 2019-2022 University of Strathclyde
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -40,6 +40,7 @@
 """Provides the report subcommand for pyani."""
 
 import logging
+import warnings
 
 from argparse import Namespace
 from typing import List, NamedTuple
@@ -154,12 +155,7 @@ def subcmd_report(args: Namespace) -> int:
             "genome label",
             "genome class",
         ]
-        report(
-            args,
-            session,
-            formats,
-            ReportParams("runs_genomes", statement, headers),
-        )
+        report(args, session, formats, ReportParams("runs_genomes", statement, headers))
 
     # Report table of all runs in which a genome is involved
     if args.show_genomes_runs:
@@ -196,12 +192,7 @@ def subcmd_report(args: Namespace) -> int:
             "method",
             "date run",
         ]
-        report(
-            args,
-            session,
-            formats,
-            ReportParams("genomes_runs", statement, headers),
-        )
+        report(args, session, formats, ReportParams("genomes_runs", statement, headers))
 
     # Report table of comparison results for the indicated runs
     if args.run_results:
@@ -234,6 +225,7 @@ def subcmd_report(args: Namespace) -> int:
                 .filter(Run.run_id == run_id)
                 .statement
             )
+            logger.debug("Results query: %s", statement)
             headers = [
                 "Comparison ID",
                 "Query ID",
@@ -300,12 +292,7 @@ def subcmd_report(args: Namespace) -> int:
     return 0
 
 
-def report(
-    args: Namespace,
-    session,
-    formats: List[str],
-    params: ReportParams,
-) -> None:
+def report(args: Namespace, session, formats: List[str], params: ReportParams) -> None:
     """Write tabular report of pyani runs from database.
 
     :param args:  Namespace of command-line arguments
@@ -319,7 +306,25 @@ def report(
     logger.debug(
         "Writing table of pyani %s from the database to %s.*", params.name, outfname
     )
-    data = pd.read_sql(params.statement, session.bind)
+
+    # With newer versions of SQLAlchemy, Pandas may throw a warning due to the composition
+    # of our statement including a Cartesian product, even though this is what we want:
+    #   SAWarning: SELECT statement has a cartesian product between FROM element(s)
+    #   "runs" and FROM element "genome_query".  Apply join condition(s) between each
+    #   element to resolve.
+    # We could use SQLAlchemy's true() function to force the join condition, but this has
+    # to be done from within Pandas, and is an issue for them to fix.
+    # We suppress the warning, instead.
+    with warnings.catch_warnings():
+        warnings.filterwarnings(
+            "ignore",
+            message=(
+                "SELECT statement has a cartesian product between FROM "
+                'element\\(s\\) "runs" and FROM element "genome_query"'
+            ),
+        )
+        data = pd.read_sql(params.statement, session.bind)
+
     data.columns = params.headers
     pyani_report.write_dbtable(data, outfname, formats)
 
diff --git a/tests/test_subcmd_05_report.py b/tests/test_subcmd_05_report.py
index 49bf0419..d30096ce 100644
--- a/tests/test_subcmd_05_report.py
+++ b/tests/test_subcmd_05_report.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # (c) The James Hutton Institute 2017-2019
-# (c) The University of Strathclude 2019
+# (c) The University of Strathclude 2019-2022
 # Author: Leighton Pritchard
 #
 # Contact:
@@ -19,7 +19,7 @@
 # The MIT License
 #
 # Copyright (c) 2017-2018 The James Hutton Institute
-# (c) The University of Strathclude 2019
+# (c) The University of Strathclude 2019-2022
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

From 34143487b1a00c1362f65e94374e364dcffc9032 Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Thu, 20 Jan 2022 16:16:38 +0000
Subject: [PATCH 32/33] suppress pandas/SQL SAWarnings about Cartesian joins

---
 pyani/scripts/subcommands/subcmd_report.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/pyani/scripts/subcommands/subcmd_report.py b/pyani/scripts/subcommands/subcmd_report.py
index c9d74a78..453b29cc 100644
--- a/pyani/scripts/subcommands/subcmd_report.py
+++ b/pyani/scripts/subcommands/subcmd_report.py
@@ -323,6 +323,28 @@ def report(args: Namespace, session, formats: List[str], params: ReportParams) -
                 'element\\(s\\) "runs" and FROM element "genome_query"'
             ),
         )
+        warnings.filterwarnings(
+            "ignore",
+            message=(
+                "SELECT statement has a cartesian product between FROM "
+                'element\\(s\\) "runs" and FROM element "comparisons"'
+            ),
+        )
+        warnings.filterwarnings(
+            "ignore",
+            message=(
+                "SELECT statement has a cartesian product between FROM "
+                'element\\(s\\) "runs" and FROM element "genome_subject"'
+            ),
+        )
+        warnings.filterwarnings(
+            "ignore",
+            message=(
+                "SELECT statement has a cartesian product between FROM "
+                'element\\(s\\) "genome_query", "genome_subject", "comparisons" '
+                'and FROM element "runs"'
+            ),
+        )
         data = pd.read_sql(params.statement, session.bind)
 
     data.columns = params.headers

From 7f7676700ffb001adf485f5f628f1789e40f78ed Mon Sep 17 00:00:00 2001
From: widdowquinn <leightonpritchard@mac.com>
Date: Thu, 20 Jan 2022 18:29:21 +0000
Subject: [PATCH 33/33] make plt.close() consistent across all plots

This is functionally useless, I think.
---
 pyani/scripts/subcommands/subcmd_plot.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pyani/scripts/subcommands/subcmd_plot.py b/pyani/scripts/subcommands/subcmd_plot.py
index d3682d1d..4e3f1ab0 100644
--- a/pyani/scripts/subcommands/subcmd_plot.py
+++ b/pyani/scripts/subcommands/subcmd_plot.py
@@ -194,6 +194,9 @@ def write_distribution(
             title=f"matrix_{matdata.name}_run{run_id}",
         )
 
+    # Be tidy with matplotlib caches
+    plt.close("all")
+
 
 def write_heatmap(
     run_id: int,