Merge pull request #24 from lirmm/bugfix/files_inputs

Bugfix/files inputs

docs/dev_doc/sample_code.rst

@@ -102,6 +102,43 @@ Now, you just render this form into your template (ex. in a django tpl).
     {% endblock footer %}
 
 
+Integrate a WAVES service form
+------------------------------
+
+You've got a website and you want your visitors to be able to submit jobs?
+The best way to achieve this is to add in your website WAVES forms using the dedicated API entry points
+Here, you're supposed to know there is a service named "sample_service" defined on demo WAVES instance (as you see in serviceList above).
+
+.. code-block:: python
+    from coreapi import Client, auth
+    from coreapi.codecs import CoreJSONCodec, JSONCodec, TextCodec
+    decoders = [JSONCodec(), CoreJSONCodec(), TextCodec()]
+    client = Client(decoders=decoders)
+    document = client.get('http://waves.demo.atgc-montpellier.fr/waves/api/schema')
+    wavesform = client.action(document, ['services', 'form'], params={"service_app_name": 'sample_service'}, validate=False, encoding='multipart/form-data')
+
+Now, you just render this form into your template (ex. in a django tpl).
+
+.. warning::
+    Don't forget to add forms.css and services.js from your waves instance as in this sample.
+
+.. code-block:: django
+    {% block head %}
+       {% addtoblock "waves_forms_css" %} <link rel="stylesheet" href="http://waves.demo.atgc-montpellier.fr/static/waves/css/forms.css">{% endaddtoblock %}
+    {% endblock %}
+
+    {% block main %}
+    <!-- Import the web form as is -->
+    {{ wavesform|safe }}
+    {% endblock main %}
+
+    {% block footer %}
+        {% addtoblock "js" %}
+            <script src="http://waves.demo.atgc-montpellier.fr/static/waves/js/services.js"></script>
+        {% endaddtoblock %}
+    {% endblock footer %}
+
+
 Create a job
 ------------
 

waves/wcore/adaptors/saga_python.py

@@ -135,7 +135,7 @@ def _job_status(self, job):
     def _job_description(self, job):
         desc = dict(working_directory=job.working_dir,
                     executable=self.command,
-                    arguments=job.command_line,
+                    arguments=job.command_line_arguments,
                     output=job.stdout,
                     error=job.stderr)
         return desc

waves/wcore/adaptors/tests.py

@@ -8,7 +8,7 @@
 
 from waves.wcore.adaptors.cluster import SshClusterAdaptor
 from waves.wcore.adaptors.const import JobStatus
-from waves.wcore.adaptors.exceptions import AdaptorException, AdaptorNotAvailableException
+from waves.wcore.adaptors.exceptions import AdaptorException
 from waves.wcore.adaptors.mocks import MockJobRunnerAdaptor
 from waves.wcore.adaptors.shell import LocalShellAdaptor, SshShellAdaptor, SshKeyShellAdaptor
 from waves.wcore.exceptions.jobs import JobInconsistentStateError
@@ -122,40 +122,6 @@ def test_credentials(self):
             except AdaptorException as e:
                 logger.warning("AdaptorException in %s: %s", adaptor.__class__.__name__, e.message)
 
-    def test_local_cp_job(self):
-        adaptor = self.adaptors['local']
-        logger.debug('Connecting to %s', adaptor.name)
-        job = self.create_cp_job(source_file=self.get_sample(),
-                                 submission=self.create_random_service().default_submission)
-        job.adaptor = adaptor
-        logger.info('job command line %s ', job.command_line)
-        self.run_job_workflow(job)
-
-    def test_slurm_cp_job(self):
-        if self.adaptors.has_key("sshSlurm"):
-            adaptor = self.adaptors["sshSlurm"]
-            logger.debug('Connecting to %s', adaptor.name)
-            job = self.create_cp_job(source_file=self.get_sample(),
-                                     submission=self.create_random_service().default_submission)
-            job.adaptor = adaptor
-            logger.info('job command line %s ', job.command_line)
-            self.run_job_workflow(job)
-        else:
-            self.skipTest("No Slurm configured")
-
-    def test_all_cp_jobs(self):
-        for name, adaptor in self.adaptors.items():
-            try:
-                logger.debug('Connecting to %s', adaptor.name)
-                job = self.create_cp_job(source_file=self.get_sample(),
-                                         submission=self.create_random_service().default_submission)
-                adaptor.command = 'cp'
-                job.adaptor = adaptor
-                self.run_job_workflow(job)
-            except AdaptorNotAvailableException as e:
-                logger.info('Adaptor %s is not locally available', adaptor)
-                logger.exception(e.message)
-
     def debug_job_state(self):
         logger.debug('Internal state %s, current %s', self.current_job._status, self.current_job.status)
 
@@ -193,3 +159,6 @@ def test_job_states(self):
         self.current_job.status = JobStatus.JOB_RUNNING
         self.current_job.run_cancel()
         self.assertTrue(self.current_job.status == JobStatus.JOB_CANCELLED)
+        self.current_job.delete()
+
+

waves/wcore/admin/inputs.py

@@ -196,7 +196,9 @@ class FileInputAdmin(AParamAdmin):
     show_in_index = False
     extra_fieldset_title = 'File params'
 
-    inlines = [FileInputSampleInline, SampleDependentInputInline]
+    inlines = [FileInputSampleInline]
+    # FIX ME Sample dependent input are not managed so far.
+    # inlines = [FileInputSampleInline, SampleDependentInputInline]
     fieldsets = [
         ('General', {
             'fields': required_base_fields + ['allow_copy_paste', 'max_size', 'allowed_extensions'],

waves/wcore/admin/jobs.py

@@ -113,7 +113,7 @@ class JobAdmin(WavesModelAdmin):
     list_per_page = 30
     search_fields = ('client__email', 'get_run_on')
     readonly_fields = ('title', 'slug', 'submission_service_name', 'email_to', '_status', 'created', 'updated',
-                       'get_run_on', 'command_line', 'remote_job_id', 'submission_name', 'nb_retry',
+                       'get_run_on', 'command_line_arguments', 'remote_job_id', 'submission_name', 'nb_retry',
                        'connexion_string', 'get_command_line', 'working_dir', 'exit_code', 'get_run_details')
 
     fieldsets = [
@@ -223,9 +223,9 @@ def connexion_string(self, obj):
         else:
             return "Unavailable"
 
-    def get_command_line(self, obj):
-        if obj.adaptor:
-            return "%s %s" % (obj.adaptor.command, obj.command_line)
+    def get_command_line(self, job):
+        if job.adaptor:
+            return "%s %s" % (job.adaptor.command, job.command_line_arguments)
         else:
             return "Unavailable"
 

waves/wcore/admin/services.py

@@ -144,6 +144,13 @@ def add_view(self, request, form_url='', extra_context=None):
         context['show_save'] = False
         return super(ServiceAdmin, self).add_view(request, form_url, extra_context=context)
 
+    def change_view(self, request, object_id, form_url='', extra_context=None):
+        """ On the change view, disable 'Save as new' and 'Save and add another' Django  default buttons """
+        context = extra_context or {}
+        context['show_save_as_new'] = False
+        context['show_save_and_add_another'] = False
+        return super(ServiceAdmin, self).change_view(request, object_id, form_url, context)
+
     def submission_link(self, obj):
         """ Direct link to submission in list """
         links = []

waves/wcore/admin/submissions.py

@@ -249,7 +249,7 @@ def get_name(self, obj):
     def get_command_line_pattern(self, obj):
         if not obj.adaptor:
             return "N/A"
-        return "%s %s" % (obj.adaptor.command, obj.service.command.create_command_line(inputs=obj.inputs.all()))
+        return "%s %s" % (obj.adaptor.command, obj.service.command_parser.create_command_line(inputs=obj.inputs.all()))
 
     def has_add_permission(self, request):
         return False

waves/wcore/api/tests.py

@@ -103,7 +103,7 @@ def test_create_job_api(self):
                 for job_input in submission['inputs']:
                     if job_input['type'] == ParamType.TYPE_FILE:
                         i += 1
-                        input_data = self.get_test_file(job_input['name'], i)
+                        input_data = self.get_test_file()
                         logger.debug('file input %s', input_data)
                     elif job_input['type'] == ParamType.TYPE_INT:
                         input_data = int(random.randint(0, 199))
@@ -133,14 +133,18 @@ def test_create_job_api(self):
                 self.assertEqual(response.status_code, status.HTTP_201_CREATED)
                 job = Job.objects.all().order_by('-created').first()
                 logger.debug(job)
+
+        self.assertEqual(expected_jobs, Job.objects.count())
+
         for job in Job.objects.all():
             logger.debug('Job %s ', job)
             self.assertEqual(job.status, JobStatus.JOB_CREATED)
             self.assertEqual(job.job_history.count(), 1)
             self.assertIsNotNone(job.job_inputs)
             self.assertIsNotNone(job.outputs)
             self.assertGreaterEqual(job.outputs.count(), 2)
-        self.assertEqual(expected_jobs, Job.objects.count())
+            job.delete()
+
 
     def test_update_job(self):
         pass
@@ -218,14 +222,15 @@ def test_create_job_api(self):
                 job = Job.objects.get(slug=response.data['slug'])
                 self.assertEqual(job.email_to, 'wavesapi@waves.wcore.fr')
                 logger.debug(job)
+        self.assertEqual(expected_jobs, Job.objects.count())
         for job in Job.objects.all():
             logger.debug('Job %s ', job)
             self.assertEqual(job.status, JobStatus.JOB_CREATED)
             self.assertEqual(job.job_history.count(), 1)
             self.assertIsNotNone(job.job_inputs)
             self.assertIsNotNone(job.outputs)
             self.assertGreaterEqual(job.outputs.count(), 2)
-        self.assertEqual(expected_jobs, Job.objects.count())
+            job.delete()
 
     def test_cancel_job(self):
         """
@@ -244,6 +249,7 @@ def test_cancel_job(self):
         self.assertEqual(db_job.status, JobStatus.JOB_CANCELLED)
         test_cancel = self.client.put(reverse('wapi:v2:waves-jobs-cancel', kwargs={'unique_id': sample_job.slug}))
         self.assertEqual(test_cancel.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
+        db_job.delete()
 
     def test_delete_job(self):
         runner = Runner.objects.create(name="Mock Runner",

waves/wcore/api/v2/serializers/inputs.py

@@ -18,12 +18,16 @@ class Meta:
     description = serializers.CharField(source='help_text')
     edam_formats = CommaSeparatedListField()
     edam_datas = CommaSeparatedListField()
-    dependents_inputs = RecursiveField(many=True, read_only=True)
 
     def to_representation(self, instance):
         repr_initial = super(AParamSerializer, self).to_representation(instance)
         if instance.dependents_inputs.count() == 0:
             repr_initial.pop('dependents_inputs')
+        else:
+            repr_initial['dependents_inputs'] = {}
+            for dep_input in instance.dependents_inputs.all():
+                serializer = InputSerializer(dep_input)
+                repr_initial['dependents_inputs'].update({dep_input.api_name: serializer.to_representation(dep_input)})
         if instance.when_value is None:
             repr_initial.pop('when_value')
         return repr_initial
@@ -73,7 +77,7 @@ class Meta(AParamSerializer.Meta):
         fields = AParamSerializer.Meta.fields + ['max_size', 'allowed_extensions']
 
 
-class ListSerialzer(AParamSerializer):
+class ListSerializer(AParamSerializer):
     class Meta(AParamSerializer.Meta):
         model = ListParam
         fields = AParamSerializer.Meta.fields + ['values_list']
@@ -100,7 +104,7 @@ def to_representation(self, obj):
         if isinstance(obj, FileInput):
             return FileSerializer(obj, context=self.context).to_representation(obj)
         elif isinstance(obj, ListParam):
-            return ListSerialzer(obj, context=self.context).to_representation(obj)
+            return ListSerializer(obj, context=self.context).to_representation(obj)
         elif isinstance(obj, BooleanParam):
             return BooleanSerializer(obj, context=self.context).to_representation(obj)
         elif isinstance(obj, IntegerParam):