New model Interpreter. Used in Server and Job.

.env.base

@@ -21,3 +21,9 @@ TEST_REMOTE_DIRECTORY=
 
 # The filename to store the scripts as
 TEST_REMOTE_FILENAME=
+
+# The interpreter name that will run the script
+TEST_INTERPRETER_NAME=
+
+# The interpreter path that will run the script
+TEST_INTERPRETER_PATH=

django_remote_submission/admin.py

@@ -5,13 +5,16 @@
 from django.shortcuts import render
 from django.http.response import HttpResponseRedirect
 
-from .models import Server, Job, Log
+from .models import Server, Job, Log, Interpreter
 from .tasks import submit_job_to_server
 
+@admin.register(Interpreter)
+class InterpreterAdmin(admin.ModelAdmin):
+    pass
 
 @admin.register(Server)
 class ServerAdmin(admin.ModelAdmin):
-    pass
+    filter_horizontal = ('interpreters',)
 
 
 @admin.register(Job)

django_remote_submission/migrations/0001_initial.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.10.4 on 2016-12-06 23:11
+# Generated by Django 1.10.4 on 2016-12-08 17:56
 from __future__ import unicode_literals
 
 from django.conf import settings
@@ -18,6 +18,20 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        migrations.CreateModel(
+            name='Interpreter',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
+                ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
+                ('name', models.CharField(help_text='The human-readable name of the interpreter', max_length=100, verbose_name='Interpreter Name')),
+                ('path', models.CharField(help_text='The full path of the interpreter path and additional parameters.', max_length=256, verbose_name='Command Full Path')),
+            ],
+            options={
+                'verbose_name': 'interpreter',
+                'verbose_name_plural': 'interpreters',
+            },
+        ),
         migrations.CreateModel(
             name='Job',
             fields=[
@@ -29,6 +43,7 @@ class Migration(migrations.Migration):
                 ('status', model_utils.fields.StatusField(choices=[('initial', 'initial'), ('submitted', 'submitted'), ('success', 'success'), ('failure', 'failure')], default='initial', help_text='The current status of the program', max_length=100, no_check_for_status=True, verbose_name='Job Status')),
                 ('remote_directory', models.CharField(help_text='The directory on the remote host to store the program', max_length=250, verbose_name='Job Remote Directory')),
                 ('remote_filename', models.CharField(help_text='The filename to store the program to (e.g. reduce.py)', max_length=250, verbose_name='Job Remote Filename')),
+                ('interpreter', models.ForeignKey(help_text='The interpreter that this job will run on', on_delete=django.db.models.deletion.PROTECT, related_name='jobs', to='django_remote_submission.Interpreter', verbose_name='Job Interpreter')),
                 ('owner', models.ForeignKey(help_text='The user that owns this job', on_delete=django.db.models.deletion.PROTECT, related_name='jobs', to=settings.AUTH_USER_MODEL, verbose_name='Job Owner')),
             ],
             options={
@@ -59,6 +74,7 @@ class Migration(migrations.Migration):
                 ('title', models.CharField(help_text='The human-readable name of the server', max_length=100, verbose_name='Server Name')),
                 ('hostname', models.CharField(help_text='The hostname used to connect to the server', max_length=100, verbose_name='Server Hostname')),
                 ('port', models.IntegerField(default=22, help_text='The port to connect to for SSH (usually 22)', verbose_name='Server Port')),
+                ('interpreters', models.ManyToManyField(to='django_remote_submission.Interpreter', verbose_name='List of interpreters available for this Server')),
             ],
             options={
                 'verbose_name': 'server',

django_remote_submission/models.py

@@ -4,11 +4,32 @@
 from django.utils.translation import ugettext_lazy as _
 from django.conf import settings
 from django.utils.encoding import python_2_unicode_compatible
+from django.core.exceptions import ValidationError
 
 from model_utils import Choices
 from model_utils.fields import StatusField, AutoCreatedField
 from model_utils.models import TimeStampedModel
 
+@python_2_unicode_compatible
+class Interpreter(TimeStampedModel):
+    name = models.CharField(
+        _('Interpreter Name'),
+        help_text=_('The human-readable name of the interpreter'),
+        max_length=100,
+    )
+
+    path = models.CharField(
+        _('Command Full Path'),
+        help_text=_('The full path of the interpreter path and additional parameters.'),
+        max_length=256,
+    )
+
+    class Meta:
+        verbose_name = _('interpreter')
+        verbose_name_plural = _('interpreters')
+
+    def __str__(self):
+        return '{self.name} ({self.path})'.format(self=self)
 
 @python_2_unicode_compatible
 class Server(TimeStampedModel):
@@ -30,6 +51,11 @@ class Server(TimeStampedModel):
         default=22,
     )
 
+    interpreters = models.ManyToManyField(
+        Interpreter,
+        verbose_name=_("List of interpreters available for this Server")
+    )
+
     class Meta:
         verbose_name = _('server')
         verbose_name_plural = _('servers')
@@ -91,12 +117,34 @@ class Job(TimeStampedModel):
         help_text=_('The server that this job will run on'),
     )
 
+    interpreter = models.ForeignKey(
+        Interpreter,
+        models.PROTECT,
+        related_name='jobs',
+        verbose_name=_('Job Interpreter'),
+        help_text=_('The interpreter that this job will run on'),
+    )
+
     class Meta:
         verbose_name = _('job')
         verbose_name_plural = _('jobs')
 
     def __str__(self):
         return '{self.title}'.format(self=self)
+
+    def clean(self):
+        '''
+        Makes sure the interpreter exists for this Server
+        This only works for the form job creation!
+        TODO: Put this in the pre_save signal
+        '''
+        available_interpreters = self.server.interpreters.all()
+        if self.interpreter not in available_interpreters:
+            raise ValidationError(_('The Interpreter picked is not valid for this server. '))
+            #'Please, choose one from: {0!s}.').format(available_interpreters))
+        else:
+            cleaned_data = super(Job, self).clean()
+            return cleaned_data
 
 
 @python_2_unicode_compatible

django_remote_submission/tasks.py

@@ -1,5 +1,4 @@
 from __future__ import absolute_import, unicode_literals, print_function
-from six import with_metaclass
 
 import logging
 
@@ -14,7 +13,7 @@
 from paramiko.client import SSHClient, AutoAddPolicy
 from threading import Thread
 
-from .models import Log, Job
+from .models import Log, Job, Interpreter
 
 try:
     from celery import shared_task
@@ -35,51 +34,6 @@ class LogPolicy(object):
     LOG_LIVE = 1
     LOG_TOTAL = 2
 
-
-class CommandMeta(type):
-    '''
-    Commands available in analysis.sns.gov
-    (except for python3!)
-    '''
-    defaults = {
-        'python' : '/usr/bin/python -u',
-        'python2' : '/usr/bin/python2 -u',
-        'python2.7' : '/usr/bin/python2.7 -u',
-        'python3' : '/usr/bin/python3 -u',
-        'python3.4' : '/usr/bin/python3.4 -u',
-        'python3.5' : '/usr/bin/python3.5 -u',
-        'bash' : '/usr/bin/bash',
-        'sh' : '/bin/sh',
-        'mantidpython' : '/usr/bin/mantidpython38 -u',
-        'mantidpython35' : '/usr/bin/mantidpython35 -u',
-        'mantidpython36' : '/usr/bin/mantidpython36 -u',
-        'mantidpython37' : '/usr/bin/mantidpython37 -u',
-        'mantidpython38' : '/usr/bin/mantidpython38 -u',
-        'mantidpythonnightly' : '/usr/bin/mantidpythonnightly -u',
-    }
-    def __getitem__(cls,key):
-        '''
-        The default of get item is the key if val not found
-        '''
-        val = cls.defaults.get(key,key)
-        return val
-
-class Command(with_metaclass(CommandMeta)):
-    '''
-    In the code just call
-        In [10]: Command['python']
-        Out[10]: '/usr/bin/python -u'
-    or:
-        In [4]: Command.build_command('python',60)
-        Out[4]: 'timeout 60s /usr/bin/python -u XX'
-    '''
-    @staticmethod
-    def build_command(interpreter, timeout, job):
-        command = '{} {}'.format(Command[interpreter],job.remote_filename)
-        if timeout is not None:
-            command = 'timeout {}s {}'.format(timeout.total_seconds(), command)
-        return command
-
 def store_logs(stream, stream_type, log_policy, job):
     '''
     Store logs in the DB
@@ -114,8 +68,7 @@ def store_logs(stream, stream_type, log_policy, job):
 
 @shared_task
 def submit_job_to_server(job_pk, password, username=None, client=None,
-                         log_policy=LogPolicy.LOG_LIVE, timeout=None,
-                         interpreter = 'python'):
+                         log_policy=LogPolicy.LOG_LIVE, timeout=None):
     job = Job.objects.get(pk=job_pk)
 
     if username is None:
@@ -138,8 +91,12 @@ def submit_job_to_server(job_pk, password, username=None, client=None,
     job.status = Job.STATUS.submitted
     job.save()
 
-    command = Command.build_command(interpreter,timeout, job)
-    logger.debug("Executing remotely teh command: %s.", command)
+    command = '{} {}'.format(job.interpreter.path,job.remote_filename)
+    if timeout is not None:
+        command = 'timeout {}s {}'.format(timeout.total_seconds(), command)
+    return command
+
+    logger.debug("Executing remotely the command: %s.", command)
 
     stdin, stdout, stderr = client.exec_command(
         command='cd {} && {}'.format(
@@ -153,7 +110,7 @@ def submit_job_to_server(job_pk, password, username=None, client=None,
 
     # In parallel creates logs for both stdout and stderr
     params = [(stdout,'stdout', log_policy, job),
-              (stderr,'stderr',log_policy, job)
+              (stderr,'stderr', log_policy, job)
              ]
     threads = []
     for param in params: