Auto-detect no. of available cores with -j 0

Closes #322

CHANGES.rst

@@ -5,11 +5,14 @@ Changes
 Development version
 -------------------
 
-* Close :issue:`225`: Allow setting the maximum error rate and
-  minimum overlap length per adapter. A new
-  :ref:`syntax for adapter-specific
-  parameters <trimming-parameters>` was added for this.
-  Example: ``-a "ADAPTER;min_overlap=5"``.
+* Close :issue:`322`: Use ``-j 0`` to auto-detect how many cores to run on.
+  This should even work correctly on cluster systems when Cutadapt runs as
+  a batch job to which fewer cores than exist on the machine. Note that
+  the number of threads used by ``pigz`` cannot be controlled at the moment,
+  see :issue:`290`.
+* Close :issue:`225`: Allow setting the maximum error rate and minimum overlap
+  length per adapter. A new :ref:`syntax for adapter-specific
+  parameters <trimming-parameters>` was added for this. Example: ``-a "ADAPTER;min_overlap=5"``.
 * Close :issue:`152`: Using the new syntax for adapter-specific parameters,
   it is now possible to allow partial matches of a 3' adapter at the 5' end
   (and partial matches of a 5' adapter at the 3' end) by specifying the

doc/guide.rst

@@ -115,22 +115,22 @@ Multi-core support
 ------------------
 
 Cutadapt supports parallel processing, that is, it can use multiple CPU cores.
-Multi-core is currently not enabled by default. To enable it, use the
-option ``-j N`` (or the spelled-out version ``--cores=N``), where ``N`` is the
+Multi-core is not enabled by default. To enable it, use the option ``-j N``
+(or the spelled-out version ``--cores=N``), where ``N`` is the
 number of cores to use.
 
+To automatically detect the number of available cores, use ``-j 0``
+(or ``--cores=0``). The detection takes into account resource restrictions
+that may be in place. For example, if running Cutadapt as a batch job on a
+cluster system, the actual number of cores assigned to the job will be used.
+(This works if the cluster systems uses the cpuset(1) mechanism to impose
+the resource limitation.)
+
 Make also sure that you have ``pigz`` (parallel gzip) installed if you use
 multiple cores and write to a ``.gz`` output file. Otherwise, compression of
-the output will be done in a single thread and therefore be the main bottleneck.
-
-.. note::
-    In a future release, the plan is to make cutadapt automatically use as many
-    CPU cores as are available, even when no ``--cores`` option was given.
-    Please help to ensure that multi-core support is as stable as possible by
-    `reporting any problems <https://github.com/marcelm/cutadapt/issues>`_ you
-    may find!
+the output will be done in a single thread and therefore be a bottleneck.
 
-There are some limitations:
+There are some limitations at the moment:
 
 * Multi-core is *only* available when you run cutadapt with Python 3.3 or later.
 * Multi-core cutadapt can only write to output files given by ``-o`` and ``-p``.
@@ -159,6 +159,8 @@ Some of these limitations will be lifted in the future, as time allows.
 
 .. versionadded:: 1.15
 
+.. versionadded:: 1.18
+    ``--cores=0`` for autodetection
 
 Read processing stages
 ======================

src/cutadapt/__main__.py

@@ -71,8 +71,9 @@
 from cutadapt.modifiers import (LengthTagModifier, SuffixRemover, PrefixSuffixAdder,
 	DoubleEncoder, ZeroCapper, PrimerTrimmer, QualityTrimmer, UnconditionalCutter,
 	NEndTrimmer, AdapterCutter, NextseqQualityTrimmer, Shortener)
-from cutadapt.report import Statistics, print_report, print_minimal_report, redirect_standard_output
-from cutadapt.pipeline import SingleEndPipeline, PairedEndPipeline, OutputFiles, ParallelPipelineRunner, available_cpu_count
+from cutadapt.report import print_report, print_minimal_report, redirect_standard_output
+from cutadapt.pipeline import SingleEndPipeline, PairedEndPipeline, OutputFiles, ParallelPipelineRunner
+from cutadapt.utils import available_cpu_count
 from cutadapt.compat import PY3
 
 logger = logging.getLogger()
@@ -123,7 +124,7 @@ def get_option_parser():
 			"Ignored when reading csfasta/qual files. Default: auto-detect "
 			"from file name extension.")
 	parser.add_option('-j', '--cores', type=int, default=1,
-		help='Number of CPU cores to use. Default: %default')
+		help='Number of CPU cores to use. Use 0 to auto-detect. Default: %default')
 
 	# Hidden options
 	parser.add_option("--gc-content", type=float, default=50,  # it's a percentage
@@ -697,7 +698,9 @@ def main(cmdlineargs=None, default_outfile=sys.stdout):
 		parser.error(e)
 		return  # avoid IDE warnings below
 
-	cores = max(1, options.cores)
+	if options.cores < 0:
+		parser.error('Value for --cores cannot be negative')
+	cores = available_cpu_count() if options.cores == 0 else options.cores
 	if cores > 1:
 		if (
 			PY3

src/cutadapt/pipeline.py

@@ -2,7 +2,6 @@
 
 import io
 import os
-import re
 import sys
 import copy
 import logging
@@ -326,29 +325,6 @@ def _create_demultiplexer(self, outfiles):
 			colorspace=self._colorspace)
 
 
-def available_cpu_count():
-	"""
-	Return the number of available virtual or physical CPUs on this system.
-	The number of available CPUs can be smaller than the total number of CPUs
-	when the cpuset(7) mechanism is in use, as is the case on some cluster
-	systems.
-
-	Adapted from http://stackoverflow.com/a/1006301/715090
-	"""
-	try:
-		with open('/proc/self/status') as f:
-			status = f.read()
-		m = re.search(r'(?m)^Cpus_allowed:\s*(.*)$', status)
-		if m:
-			res = bin(int(m.group(1).replace(',', ''), 16)).count('1')
-			if res > 0:
-				return min(res, multiprocessing.cpu_count())
-	except IOError:
-		pass
-
-	return multiprocessing.cpu_count()
-
-
 def reader_process(file, file2, connections, queue, buffer_size, stdin_fd):
 	"""
 	Read chunks of FASTA or FASTQ data from *file* and send to a worker.
@@ -514,7 +490,7 @@ def wrote_everything(self):
 
 class ParallelPipelineRunner(object):
 	"""
-	Wrap a SingleEndPipeline, running it in parallel
+	Run a Pipeline in parallel
 
 	- When set_input() is called, a reader process is spawned.
 	- When run() is called, as many worker processes as requested are spawned.

src/cutadapt/utils.py

@@ -0,0 +1,25 @@
+import multiprocessing
+import re
+
+
+def available_cpu_count():
+	"""
+	Return the number of available virtual or physical CPUs on this system.
+	The number of available CPUs can be smaller than the total number of CPUs
+	when the cpuset(7) mechanism is in use, as is the case on some cluster
+	systems.
+
+	Adapted from http://stackoverflow.com/a/1006301/715090
+	"""
+	try:
+		with open('/proc/self/status') as f:
+			status = f.read()
+		m = re.search(r'(?m)^Cpus_allowed:\s*(.*)$', status)
+		if m:
+			res = bin(int(m.group(1).replace(',', ''), 16)).count('1')
+			if res > 0:
+				return min(res, multiprocessing.cpu_count())
+	except IOError:
+		pass
+
+	return multiprocessing.cpu_count()

tests/test_commandline.py

@@ -13,7 +13,7 @@
 import pytest
 
 from cutadapt.__main__ import main
-from cutadapt.compat import StringIO
+from cutadapt.compat import StringIO, PY3
 from utils import run, assert_files_equal, datapath, cutpath, redirect_stderr, temporary_path
 
 import pytest_timeout as _unused
@@ -543,3 +543,9 @@ def test_discard_casava():
 def test_underscore():
 	"""File name ending in _fastq.gz (issue #275)"""
 	run('-b TTAGACATATCTCCGTCG', 'small.fastq', 'underscore_fastq.gz')
+
+
+if PY3:
+	def test_cores_autodetect():
+		# Just make sure that it runs; functionality is not tested
+		run('--cores 0 -b TTAGACATATCTCCGTCG', 'small.fastq', 'underscore_fastq.gz')