Merge branch 'dev' into feature/py3

.travis.yml

@@ -1,7 +1,7 @@
 language: python
 python:
   - "2.7"
-  
+
 sudo: required
 
 services:
@@ -22,35 +22,35 @@ before_install:
 install:
   - docker-compose -f docker-compose.yml pull
   - docker-compose -f docker-compose.yml build
-  - docker-compose -f docker-compose.yml -f env/js_build.yml run js_build
+  - docker-compose -f docker-compose.yml -f env/front_build.yml run front_build
 
 script:
 # -------------DEFAULT--------------------
   - if [[ $TEST_TARGET == 'default' ]]; then
-      docker-compose -f docker-compose.yml -f env/coverage.yml run app  sh -c "cd /srv/src/timeside;  pip install -U coverage; coverage run --source=timeside --omit=timeside/plugins/analyzer/external/yaafe.py setup.py test";
+      docker-compose -f docker-compose.yml -f env/coverage.yml run app  sh -c "cd /srv/lib/timeside;  pip install -U coverage; coverage run --source=timeside --omit=timeside/plugins/analyzer/external/yaafe.py setup.py test";
     fi
 # -------------DOC------------------------
   - if [[ $TEST_TARGET == 'doc' ]]; then
-       docker-compose -f docker-compose.yml run app  sh -c "cd /srv/src/timeside/docs; pip install sphinx numpydoc; make html; make doctest";
+       docker-compose -f docker-compose.yml run app  sh -c "cd /srv/lib/timeside/docs; pip install sphinx numpydoc; make html; make doctest";
     fi
 #  # -------------CLI-------------------------
   - if [[ $TEST_TARGET == 'cli' ]]; then
-       docker-compose -f docker-compose.yml run app  sh -c "cd /srv/src/timeside;  ./tests/test_scripts.sh";
+       docker-compose -f docker-compose.yml run app  sh -c "cd /srv/lib/timeside;  ./tests/test_scripts.sh";
     fi
 #  # -------------SERVER-------------------------
 #  #  Get environment variables from .env file and run test
   - if [[ $TEST_TARGET == 'server' ]]; then
        docker-compose -f docker-compose.yml up -d db;
        sleep 20;
-       docker-compose -f docker-compose.yml up -d;
+       docker-compose -f docker-compose.yml -f nginx.yml up -d;
        sleep 20;
-       curl --fail --retry 10 --retry-delay 5 -v -X GET "$(docker-compose port nginx 80)/timeside/api/items/" -H "Authorization":" Token Dummy_Token";
+       docker-compose exec app python manage.py test timeside.server.tests;
     fi
 #
 after_success:
 #  -  # -------------DEFAULT--------------------
   - if [[ $TEST_TARGET == 'default' ]]; then
-          docker-compose -f docker-compose.yml -f env/coverage.yml run app  sh -c "cd /srv/src/timeside; pip install coveralls; coveralls";
+          docker-compose -f docker-compose.yml -f env/coverage.yml run app  sh -c "cd /srv/lib/timeside; pip install coveralls; coveralls";
     fi
 
 after_script:

app/bin/app_base.sh

@@ -30,6 +30,7 @@ gid='www-data'
 #pip install django-filter==1.1.0 djangorestframework==3.8 django==1.10 librosa==0.6.3
 #npm install -g bower
 pip install librosa==0.7.0
+pip install -U youtube-dl
 
 # pip install django-inspect
 

app/bin/install_vamp_plugins.sh

@@ -12,7 +12,7 @@ VAMP_DIR=/usr/lib/vamp
 #rm -r vampy-e2bb3cf7adf1
 
 # Link Python vamp plugins from Timeside
-#ln -s /srv/src/timeside/timeside/plugins/analyzer/externals/vampy/* $VAMP_DIR/
+#ln -s /srv/lib/timeside/timeside/plugins/analyzer/externals/vampy/* $VAMP_DIR/
 
 # QM Vamp Plugins
 wget https://code.soundsoftware.ac.uk/attachments/download/1602/qm-vamp-plugins-linux64-v1.7.1.tar.bz2

app/bin/notebook.sh

@@ -1,4 +1,4 @@
 # How-to use:
 # docker-compose -f docker-compose.yml -f env/notebook.yml run --service-ports notebook sh bin/notebook.sh
-cd /srv/src/timeside/docs/ipynb
+cd /srv/lib/timeside/docs/ipynb
 jupyter notebook --no-browser --ip=0.0.0.0 --allow-root

app/bin/timeside-player-v2-build.sh

@@ -0,0 +1,10 @@
+#/bin/bash
+
+npm install -g bower
+npm install -g grunt-cli
+apt install -y ruby-sass
+
+cd /srv/lib/timeside/timeside/player/static/timeside2
+
+bower install --allow-root
+grunt build --force

app/bin/update_environment.sh

@@ -5,7 +5,7 @@
 # Please test the environment before committing changes to environment-pinned.yml
 # `python setup.py test`
 
-cd /srv/src/timeside/
+cd /srv/lib/timeside/
 conda env create --name app --force --file conda-environment.yml
 #python setup.py test
 conda env export --name app --file environment-pinned.yml

app/settings.py

@@ -96,7 +96,7 @@
     'django.contrib.staticfiles.finders.FileSystemFinder',
     'django.contrib.staticfiles.finders.AppDirectoriesFinder',
     #    'django.contrib.staticfiles.finders.DefaultStorageFinder',
-    'djangobower.finders.BowerFinder',
+    # 'djangobower.finders.BowerFinder',
 )
 
 TEMPLATES = [
@@ -216,6 +216,8 @@
 CELERY_ACCEPT_CONTENT = ['application/json']
 CELERY_ALWAYS_EAGER = env('CELERY_ALWAYS_EAGER')  # If this is True, all tasks will be executed locally by blocking until the task returns.
 
+# TEST_RUNNER = 'djcelery.contrib.test_runner.CeleryTestSuiteRunner'
+
 from worker import app
 
 BOWER_COMPONENTS_ROOT = '/srv/static/'

docker-compose.yml

@@ -54,7 +54,6 @@ app:
   volumes:
     - ./app:/srv/app
     - ./timeside:/srv/lib/timeside/timeside
-    - ./timeside:/srv/src/timeside/timeside
     - ./lib/plugins:/srv/lib/plugins
     - ./etc/apt/sources.list:/etc/apt/sources.list
   volumes_from:

docs/source/development.rst

@@ -22,7 +22,7 @@ If the TimeSide library gives you everything you need to develop you own plugin,
 
 Before coding, start docker with mounting the local directory as a volume::
 
-    docker run -it -v .:/srv/src/timeside parisson/timeside:latest ipython
+    docker run -it -v .:/srv/lib/timeside parisson/timeside:latest ipython
 
 or use the development composition to start a notebook or the webserver::
 
@@ -35,7 +35,7 @@ If the (already huge) python module bundle provided by TimeSide is to short for
 
     git clone https://github.com/Parisson/TimeSide-Dummy.git
     cd TimeSide-Dummy
-    docker run -it -v ./timeside/plugins/:/srv/src/timeside/timeside/plugins parisson/timeside:latest ipython
+    docker run -it -v ./timeside/plugins/:/srv/lib/timeside/timeside/plugins parisson/timeside:latest ipython
 
 or::
 

docs/source/tutorials/data_management.rst

@@ -13,7 +13,7 @@ Four different *time_mode* can be specified :
 
 - 'framewise' : data are returned on a frame basis (i.e. with specified blocksize, stepsize and framerate)
 - 'global' : a global data value is return for the entire audio item
-- 'segment' : data are returned on a segmnet basis (i.e. with specified start time and duration)
+- 'segment' : data are returned on a segment basis (i.e. with specified start time and duration)
 - 'event' :  data are returned on a segment basis (i.e. with specified start time)
 
 Two different *data_mode* can be specified :
@@ -52,7 +52,7 @@ Four different time_mode can be specified :
 
 - 'framewise' : data are returned on a frame basis (i.e. with specified blocksize, stepsize and framerate)
 - 'global' : a global data value is return for the entire audio item
-- 'segment' : data are returned on a segmnet basis (i.e. with specified start time and duration)
+- 'segment' : data are returned on a segment basis (i.e. with specified start time and duration)
 - 'event' :  data are returned on a segment basis (i.e. with specified start time)
 
 

docs/source/tutorials/quick_start.rst

@@ -60,4 +60,3 @@ analysis and encoding:
    >>> encoders = get_processor('mp3_encoder')('sweep.mp3') | get_processor('flac_encoder')('sweep.flac')
    >>> (decoder | levels | encoders).run()
    >>> print levels.results
-   {'...': AnalyzerResult(id_metadata=IdMetadata(id='level.max', name='Level Analyzer Max', unit='dBFS', description='...', date='...', version='...', author='TimeSide', proc_uuid='...'), data_object=GlobalValueObject(value=array([ 0.]), y_value=array([], dtype=float64)), audio_metadata=AudioMetadata(uri='.../sweep.wav', start=0.0, duration=8.0, is_segment=False, sha1='...', channels=2, channelsManagement=''), parameters={}), '...': AnalyzerResult(id_metadata=IdMetadata(id='level.rms', name='Level Analyzer RMS', unit='dBFS', description='...', date='...', version='...', author='TimeSide', proc_uuid='...'), data_object=GlobalValueObject(value=array([-2.995]), y_value=array([], dtype=float64)), audio_metadata=AudioMetadata(uri='.../sweep.wav', start=0.0, duration=8.0, is_segment=False, sha1='...', channels=2, channelsManagement=''), parameters={})}

env/build_plugins.yml

@@ -31,5 +31,5 @@
 
 app:
   volumes:
-    - ./lib:/srv/src/plugins
+    - ./lib:/srv/lib/plugins
   command: /bin/bash /srv/app/bin/setup_plugins.sh

env/cached.yml

@@ -4,7 +4,7 @@ var:
   volumes:
      - ./bin:/srv/bin:cached
      - /Users/grandry/archive/WASABI/media:/srv/media:cached
-     - /Users/grandry/archive/WASABI/media/results_copy:/srv/media/results:cached
+     - /Users/grandry/archive/WASABI/media/results_copy:/srv/media/results
      - ./var/backup:/srv/backup:cached
      - ./var/static:/srv/static:cached
      - ./var/log/nginx:/var/log/nginx:cached
@@ -22,7 +22,6 @@ app:
   volumes:
     - ./app:/srv/app:cached
     - ./timeside:/srv/lib/timeside/timeside:cached
-    - ./timeside:/srv/src/timeside/timeside:cached
     - ./lib/plugins:/srv/lib/plugins:cached
     - ./etc/apt/sources.list:/etc/apt/sources.list:cached
   volumes_from:

env/coverage.yml

@@ -23,5 +23,5 @@
 
 app:
   volumes:
-    - .:/srv/src/timeside
+    - .:/srv/lib/timeside
 

env/debug.yml

@@ -27,7 +27,7 @@ app:
   volumes:
     - ./app:/srv/app
     - .:/srv/lib/timeside
-    - ./timeside:/srv/src/timeside/timeside
+    - ./timeside:/srv/lib/timeside/timeside
     - ./lib/plugins:/srv/lib/plugins
     - ./etc/apt/sources.list:/etc/apt/sources.list
   command: /bin/bash bin/app.sh --runserver
@@ -38,6 +38,6 @@ worker:
   env_file:
     - env/debug.env
   volumes:
-    - ./lib:/srv/src/plugins
-    - ./:/srv/src/timeside/
+    - ./lib:/srv/lib/plugins
+    - ./:/srv/lib/timeside/
 

env/js_build.yml → env/front_build.yml

@@ -23,11 +23,11 @@
 
 # Usage : docker-compose -f docker-compose.yml -f env/build.yml -f env/debug.yml -f env/dev.yml -f env/js_build.yml run --service-ports js_build
 
-js_build:
+front_build:
   image: andreptb/grunt
   volumes_from:
     - app
   ports:
     - "9001:9000"
     - "35729:35729"
-  command: sh -c "cd /srv/src/timeside/timeside/player/static/timeside2; npm install --silent; bower install --allow-root; grunt build --force; npm prune --production;"
+  command: sh -c "cd /srv/lib/timeside/timeside/player/static/timeside2; npm install --silent; bower install --allow-root; grunt build --force; npm prune --production;"

env/sqlite.yml

@@ -24,7 +24,7 @@
 
 app:
   env_file:
-    - anv/sqlite.env
+    - env/sqlite.env
   ports:
     - "8000:8000"
-  command: /bin/bash app/bin/app.sh --runserver
+  command: /bin/bash /srv/app/bin/app.sh --runserver

environment-pinned.yml

@@ -257,6 +257,7 @@ dependencies:
     - ply==3.11
     - psycopg2-binary
     - py-sonicvisualiser==0.3
+    - pytest==4.6.6
     - pympi-ling==1.69
     - redis==2.10.6
     - sphinx-rtd-theme==0.4.1

tests/test_decoder_utils.py

@@ -58,8 +58,9 @@ def setUp(self):
         self.expected_samplerate = 44100
         self.expected_depth = 0  # ?
 
-    def testUriFromHTTP(self):
-        "Test URI decoding from HTTP"
+    @unittest.skip("HTTPS is not supported by GStreamer 0.10")
+    def testUriFromHTTPS(self):
+        "Test URI decoding from HTTPS"
         self.source = "https://raw.githubusercontent.com/yomguy/timeside-samples/master/samples/sweep.mp3"
         self.test_exact_duration = False
         self.expected_depth = 32

timeside/player/static/timeside2/app/scripts/core/config/config.json

@@ -3,13 +3,13 @@
   "env" : "DEV",
   "envAdmin" : "DEV",
   "remotes" : {
-    "DEV" : "https://wasabi.telemeta.org/timeside",
+    "DEV" : "http://localhost:9030/timeside",
     "RECETTE" : "/timeside",
     "PROD" : "http://client-eu.syk.com",
     "ERICRECETTE" : "http://192.168.1.3:8080"
   },
   "remotesWrite" : {
-    "DEV" : "https://wasabi.telemeta.org/timeside",
+    "DEV" : "http://localhost:9030/timeside",
     "RECETTE" : "/timeside",
     "PROD" : "http://admin1.syk.com",
     "ERICRECETTE" : "http://192.168.1.3:8080"

timeside/plugins/provider/deezer.py

@@ -28,6 +28,8 @@ def get_source_from_id(self, external_id, path, download=False):
             file_name = file_name.replace(' ','_')  + '.mp3'
             file_path = os.path.join(path,file_name)
             r = get(source_uri)
+            if not os.path.exists(path):
+                os.makedirs(path)
             with open(file_path,'wb') as f:
                 f.write(r.content)
                 f.close()

timeside/plugins/provider/youtube.py

@@ -34,7 +34,8 @@ def get_source(self, url, path, download=False):
                     'postprocessors': [{'key':'FFmpegExtractAudio'}],
                     'restrictfilenames':True,
                 }
-
+        if not os.path.exists(path):
+                os.makedirs(path)
         with youtube_dl.YoutubeDL(ydl_opts) as ydl:
             info = ydl.extract_info(url, download=download)
             file_path = ydl.prepare_filename(info)            

timeside/server/migrations/0005_auto_20191104_1526.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10 on 2019-11-04 14:26
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('timeside_server', '0004_result_run_time'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='result',
+            options={'ordering': ['-date_added'], 'verbose_name': 'Result', 'verbose_name_plural': 'Results'},
+        ),
+    ]

timeside/server/models.py

@@ -194,6 +194,7 @@ class Meta:
 class Provider(Named, UUID):
 
     pid = models.CharField(_('pid'), blank=True, max_length=128)
+    source_access =  models.BooleanField(default=False)
 
     def __unicode__(self):
         return unicode(self.pid)
@@ -357,7 +358,7 @@ def run(self, experience):
                 proc = proc(**json.loads(preset.parameters))
                 if 'analyzer' in proc.parents:
                     parent_analyzers.append(proc.parents['analyzer'])
-        
+
         for preset in experience.presets.all():
             # get core audio processor corresponding to preset.processor.pid
             proc = preset.processor.get_processor()
@@ -394,7 +395,7 @@ def set_results_from_processor(proc, preset=None):
             elif preset is None:
                 processor, c = Processor.get_first_or_create(pid=proc.id())
                 parameters = json.dumps(processor.get_parameters_default())
-                preset, c = Preset.get_first_or_create(processor=processor, 
+                preset, c = Preset.get_first_or_create(processor=processor,
                                                     parameters=parameters)
             result, c = Result.objects.get_or_create(preset=preset,
                                                     item=self)
@@ -426,9 +427,9 @@ def set_results_from_processor(proc, preset=None):
                         f.close()
                         f_xml.close()
                     result.file = result_file.replace(settings.MEDIA_ROOT, '')
-                        
+
             result.run_time_setter(proc.run_time)
-            
+
             result.status_setter(_DONE)
 
         for preset, proc in presets.iteritems():
@@ -450,7 +451,7 @@ def set_results_from_processor(proc, preset=None):
                 result.run_time_setter(run_time)
                 result.mime_type_setter(get_mime_type(result.file.path))
                 result.status_setter(_DONE)
-                
+
 
                 if 'analyzer' in proc.parents:
                     analyzer = proc.parents['analyzer']
@@ -460,7 +461,7 @@ def set_results_from_processor(proc, preset=None):
                 result = Result.objects.get(preset=preset, item=self)
                 result.run_time_setter(proc.run_time)
                 result.mime_type_setter(get_mime_type(result.file.path))
-                result.status_setter(_DONE)                
+                result.status_setter(_DONE)
 
             if hasattr(proc, 'values'):
                 proc.values = None
@@ -590,6 +591,7 @@ class Result(UUID, Dated, Shareable):
     class Meta:
         verbose_name = _('Result')
         verbose_name_plural = _('Results')
+        ordering = ['-date_added']
 
     def status_setter(self, status):
         self.status = status
@@ -653,6 +655,7 @@ def run(self, wait=False, streaming=False):
             status = Task.objects.get(uuid=str(self.uuid)).status
             while (status != _DONE):
                 time.sleep(0.5)
+                print('WAITING')
                 status = Task.objects.get(uuid=str(self.uuid)).status
 
 

timeside/server/serializers.py

@@ -172,8 +172,9 @@ def get_waveform(self, obj):
         for i in range(nb_pixels):
             values = wav_res['data_object']['value'][
                 sample_values[i]:sample_values[i + 1]]
-            min_values[i] = np.min(values)
-            max_values[i] = np.max(values)
+            if values.size:
+                min_values[i] = np.min(values)
+                max_values[i] = np.max(values)
 
         return {'start': start,
                 'stop': stop,