Add Python 3 support

.travis.yml

@@ -1,7 +1,8 @@
 language: python
 python:
   - "2.7"
-  #- "3.4"
+  - "3.4"
+  - "3.5"
 
 install:
   - pip install coveralls

scripts/smart-dispatch

@@ -1,5 +1,6 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
+from __future__ import print_function
 
 import os
 import sys
@@ -101,7 +102,7 @@ def main():
 
     # Generating all the worker commands
     worker_script = pjoin(os.path.dirname(smartdispatch.__file__), 'workers', 'base_worker.py')
-    COMMAND_STRING = 'cd "{cwd}"; python2 {worker_script} "{commands_file}" "{log_folder}" '\
+    COMMAND_STRING = 'cd "{cwd}"; python {worker_script} "{commands_file}" "{log_folder}" '\
                      '1>> "{log_folder}/worker/$PBS_JOBID\"\"_worker_{{ID}}.o" '\
                      '2>> "{log_folder}/worker/$PBS_JOBID\"\"_worker_{{ID}}.e" '
     COMMAND_STRING = COMMAND_STRING.format(cwd=os.getcwd(), worker_script=worker_script, commands_file=command_manager._commands_filename, log_folder=path_job_logs)
@@ -119,19 +120,23 @@ def main():
     pbs_filenames = job_generator.write_pbs_files(path_job_commands)
 
     # Launch the jobs
-    print "## {nb_commands} command(s) will be executed in {nb_jobs} job(s) ##".format(nb_commands=nb_commands, nb_jobs=len(pbs_filenames))
-    print "Batch UID:\n{batch_uid}".format(batch_uid=jobname)
+    print("## {nb_commands} command(s) will be executed in {nb_jobs} job(s) ##".format(nb_commands=nb_commands, nb_jobs=len(pbs_filenames)))
+    print("Batch UID:\n{batch_uid}".format(batch_uid=jobname))
     if not args.doNotLaunch:
         jobs_id = []
         for pbs_filename in pbs_filenames:
             qsub_output = check_output('{launcher} {pbs_filename}'.format(launcher=LAUNCHER if args.launcher is None else args.launcher, pbs_filename=pbs_filename), shell=True)
+
+            if isinstance(qsub_output, bytes):
+                qsub_output = qsub_output.decode("utf-8")
+
             jobs_id += [qsub_output.strip()]
 
         with open_with_lock(pjoin(path_job, "jobs_id.txt"), 'a') as jobs_id_file:
             jobs_id_file.writelines(t.strftime("## %Y-%m-%d %H:%M:%S ##\n"))
             jobs_id_file.writelines("\n".join(jobs_id) + "\n")
-        print "\nJobs id:\n{jobs_id}".format(jobs_id=" ".join(jobs_id))
-    print "\nLogs, command, and jobs id related to this batch will be in:\n {smartdispatch_folder}".format(smartdispatch_folder=path_job)
+        print("\nJobs id:\n{jobs_id}".format(jobs_id=" ".join(jobs_id)))
+    print("\nLogs, command, and jobs id related to this batch will be in:\n {smartdispatch_folder}".format(smartdispatch_folder=path_job))
 
 
 def parse_arguments():
@@ -146,7 +151,8 @@ def parse_arguments():
     parser.add_argument('-c', '--coresPerCommand', type=int, required=False, help='How many cores a command needs.', default=1)
     parser.add_argument('-g', '--gpusPerCommand', type=int, required=False, help='How many gpus a command needs.', default=1)
     # parser.add_argument('-m', '--memPerCommand', type=float, required=False, help='How much memory a command needs (in Gb).')
-    parser.add_argument('-f', '--commandsFile', type=file, required=False, help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)')
+    parser.add_argument('-f', '--commandsFile', type=argparse.FileType('r'), required=False,
+                        help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)')
 
     parser.add_argument('-l', '--modules', type=str, required=False, help='List of additional modules to load.', nargs='+')
     parser.add_argument('-x', '--doNotLaunch', action='store_true', help='Creates the QSUB files without launching them.')

smartdispatch/argument_template.py

@@ -35,7 +35,7 @@ def unfold(self, match):
         start = int(groups[0])
         end = int(groups[1])
         step = 1 if groups[2] is None else int(groups[2])
-        return map(str, range(start, end, step))
+        return [str(i) for i in range(start, end, step)]
 
 
 argument_templates = build_argument_templates_dictionnary()

smartdispatch/smartdispatch.py

@@ -97,7 +97,7 @@ def unfold_command(command):
         pos = match.end()
 
     arguments.append([text[pos:]])  # Add remaining unfolded arguments
-    arguments = [map(utils.decode_escaped_characters, argvalues) for argvalues in arguments]
+    arguments = [[utils.decode_escaped_characters(v) for v in argvalues] for argvalues in arguments]
     return ["".join(argvalues) for argvalues in itertools.product(*arguments)]
 
 

smartdispatch/tests/test_filelock.py

@@ -38,9 +38,9 @@ def _test_open_with_lock(lock_func):
         time.sleep(1)
 
     stdout, stderr = process.communicate()
-    assert_equal(stdout, "")
-    assert_true("Traceback" not in stderr, msg="Unexpected error: '{}'".format(stderr))
-    assert_true("write-lock" in stderr, msg="Forcing a race condition, try increasing sleeping time above.")
+    assert_equal(stdout, b"")
+    assert_true("Traceback" not in stderr.decode(), msg="Unexpected error: '{}'".format(stderr.decode()))
+    assert_true("write-lock" in stderr.decode(), msg="Forcing a race condition, try increasing sleeping time above.")
 
     shutil.rmtree(temp_dir)  # Cleaning up.
 

smartdispatch/tests/test_smartdispatch.py

@@ -3,7 +3,7 @@
 import shutil
 import time as t
 from os.path import join as pjoin
-from StringIO import StringIO
+from io import StringIO, open
 
 import tempfile
 from nose.tools import assert_true, assert_equal
@@ -43,11 +43,11 @@ def test_get_commands_from_file():
     commands = ["command1 arg1 arg2",
                 "command2",
                 "command3 arg1 arg2 arg3 arg4"]
-    fileobj = StringIO("\n".join(commands))
+    fileobj = StringIO(u"\n".join(commands))
     assert_array_equal(smartdispatch.get_commands_from_file(fileobj), commands)
 
     # Test stripping last line if empty
-    fileobj = StringIO("\n".join(commands) + "\n")
+    fileobj = StringIO(u"\n".join(commands) + u"\n")
     assert_array_equal(smartdispatch.get_commands_from_file(fileobj), commands)
 
 

smartdispatch/utils.py

@@ -1,4 +1,7 @@
+from __future__ import print_function
+
 import re
+import binascii
 import hashlib
 import unicodedata
 import json
@@ -15,7 +18,7 @@ def print_boxed(string):
     out = u"\u250c" + box_line + u"\u2510\n"
     out += '\n'.join([u"\u2502 {} \u2502".format(line.ljust(max_len)) for line in splitted_string])
     out += u"\n\u2514" + box_line + u"\u2518"
-    print out
+    print(out)
 
 
 def yes_no_prompt(query, default=None):
@@ -35,13 +38,13 @@ def yes_no_prompt(query, default=None):
 
 def chunks(sequence, n):
     """ Yield successive n-sized chunks from sequence. """
-    for i in xrange(0, len(sequence), n):
+    for i in range(0, len(sequence), n):
         yield sequence[i:i + n]
 
 
 def generate_uid_from_string(value):
     """ Create unique identifier from a string. """
-    return hashlib.sha256(value).hexdigest()
+    return hashlib.sha256(value.encode()).hexdigest()
 
 
 def slugify(value):
@@ -54,15 +57,23 @@ def slugify(value):
     ---------
     https://github.com/django/django/blob/1.7c3/django/utils/text.py#L436
     """
-    value = unicodedata.normalize('NFKD', unicode(value, "UTF-8")).encode('ascii', 'ignore').decode('ascii')
+    # Convert `value` to Unicode so we can slugify it using the unicodedata module.
+    try:
+        value = unicode(value, "UTF-8")
+    except NameError:
+        pass  # In Python 3, all strings are already stored as Unicode.
+
+    # Replace all compatibility characters with their equivalents.
+    value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
     value = re.sub('[^\w\s-]', '', value).strip().lower()
     return str(re.sub('[-\s]+', '_', value))
 
 
 def encode_escaped_characters(text, escaping_character="\\"):
     """ Escape the escaped character using its hex representation """
     def hexify(match):
-        return "\\x{0}".format(match.group()[-1].encode("hex"))
+        # Reference: http://stackoverflow.com/questions/18298251/python-hex-values-to-convert-to-a-string-integer
+        return "\\x" + binascii.hexlify(match.group()[-1].encode()).decode()
 
     return re.sub(r"\\.", hexify, text)
 
@@ -73,7 +84,7 @@ def decode_escaped_characters(text):
         return ''
 
     def unhexify(match):
-        return match.group()[2:].decode("hex")
+        return binascii.unhexlify(match.group()[2:]).decode()
 
     return re.sub(r"\\x..", unhexify, text)
 
@@ -92,6 +103,8 @@ def detect_cluster():
     # Get server status
     try:
         output = Popen(["qstat", "-B"], stdout=PIPE).communicate()[0]
+        if isinstance(output, bytes):
+            output = output.decode("utf-8")
     except OSError:
         # If qstat is not available we assume that the cluster is unknown.
         return None

smartdispatch/workers/base_worker.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 import os

smartdispatch/workers/tests/test_base_worker.py

@@ -25,14 +25,14 @@ def setUp(self):
         self.command_manager = CommandManager(os.path.join(self._commands_dir, "commands.txt"))
         self.command_manager.set_commands_to_run(self.commands)
 
-        self.commands_uid = map(utils.generate_uid_from_string, self.commands)
+        self.commands_uid = [utils.generate_uid_from_string(c) for c in self.commands]
 
     def tearDown(self):
         shutil.rmtree(self._commands_dir)
         shutil.rmtree(self.logs_dir)
 
     def test_main(self):
-        command = ['python2', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir]
+        command = ['python', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir]
         assert_equal(call(command), 0)
         # Simulate a resume, i.e. re-run the command, the output/error should be concatenated.
         self.command_manager.set_commands_to_run(self.commands)
@@ -106,14 +106,14 @@ def test_main(self):
                 assert_equal("", logfile.read())
 
     def test_lock(self):
-        command = ['python2', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir]
+        command = ['python', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir]
 
         # Lock the commands file before running 'base_worker.py'
         with open_with_lock(self.command_manager._commands_filename, 'r+'):
             process = Popen(command, stdout=PIPE, stderr=PIPE)
             time.sleep(1)
 
         stdout, stderr = process.communicate()
-        assert_equal(stdout, "")
-        assert_true("write-lock" in stderr, msg="Forcing a race condition, try increasing sleeping time above.")
-        assert_true("Traceback" not in stderr)  # Check that there are no errors.
+        assert_equal(stdout, b"")
+        assert_true("write-lock" in stderr.decode(), msg="Forcing a race condition, try increasing sleeping time above.")
+        assert_true("Traceback" not in stderr.decode())  # Check that there are no errors.