Merge remote-tracking branch 'origin/master' into py3-run-tests-2

.gitignore

@@ -44,6 +44,7 @@ ckan_deb/DEBIAN/prerm
 
 # node.js
 node_modules/
+package-lock.json
 
 # docker
 contrib/docker/.env

ckan/cli/less.py

@@ -51,18 +51,7 @@
     name=u'less',
     short_help=u'Compile all root less documents into their CSS counterparts')
 def less():
-    command = (u'npm', u'bin')
-    process = subprocess.Popen(
-        command,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        universal_newlines=True)
-    output = process.communicate()
-    directory = output[0].strip()
-    if not directory:
-        error_shout(u'Command "{}" returned nothing. Check that npm is '
-                    u'installed.'.format(u' '.join(command)))
-    less_bin = os.path.join(directory, u'lessc')
+    command = (u'npm', u'run', u'build')
 
     public = config.get(u'ckan.base_public_folder')
 
@@ -73,23 +62,21 @@ def less():
         f = open(custom_less, u'w')
         f.write(_custom_css[color])
         f.close()
-        _compile_less(root, less_bin, color)
+        _compile_less(root, command, color)
     f = open(custom_less, u'w')
-    f.write(u'// This file is needed in order for ./bin/less to '
+    f.write(u'// This file is needed in order for `gulp build` to '
             u'compile in less 1.3.1+\n')
     f.close()
-    _compile_less(root, less_bin, u'main')
+    _compile_less(root, command, u'main')
 
 
-def _compile_less(root, less_bin, color):
+def _compile_less(root, command, color):
     click.echo(u'compile {}.css'.format(color))
-    main_less = os.path.join(root, u'less', u'main.less')
-    main_css = os.path.join(root, u'css', u'{}.css'.format(color))
-    command = (less_bin, main_less, main_css)
+    command = command + (u'--', u'-' + color)
+
     process = subprocess.Popen(
         command,
         stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        universal_newlines=True)
+        stderr=subprocess.PIPE)
     output = process.communicate()
     click.echo(output)

ckan/config/resource_formats.json

@@ -38,10 +38,13 @@
   ["KMZ", "KMZ File", "application/vnd.google-earth.kmz+xml", ["application/vnd.google-earth.kmz"]],
   ["MDB", "Access Database", "application/x-msaccess", []],
   ["MOP", "MOPAC Input format", "chemical/x-mopac-input", []],
+  ["MP3", "MPEG-1 Audio Layer-3", "audio/mpeg", ["audio/mpeg3", "audio/x-mpeg-3", "video/mpeg", "video/x-mpeg"]],
+  ["MP4", "MPEG Video File", "video/mp4", ["application/mp4"]],
   ["MrSID", "MrSID", "image/x-mrsid", []],
   ["MXD", "ESRI ArcGIS project file", "application/x-mxd", []],
   ["NetCDF", "NetCDF File", "application/netcdf", []],
   ["N3", "N3 Triples", "application/x-n3", []],
+  ["OGG", "Ogg Vorbis Audio File", "audio/ogg", []],
   ["ODB", "OpenDocument Database", "application/vnd.oasis.opendocument.database", []],
   ["ODC", "OpenDocument Chart", "application/vnd.oasis.opendocument.chart", []],
   ["ODF", "OpenDocument Math Formula", "application/vnd.oasis.opendocument.formula", []],
@@ -73,7 +76,9 @@
   ["TORRENT", "Torrent", "application/x-bittorrent", ["bittorrent"]],
   ["TSV", "Tab Separated Values File", "text/tab-separated-values", ["text/tsv"]],
   ["TXT", "Text File", "text/plain", []],
+  ["WAV", "Wave file", "audio/x-wav", ["audio/wav"]],
   ["WCS", "Web Coverage Service", "wcs", []],
+  ["WebM", "WEBM video", "video/webm", []],
   ["WFS", "Web Feature Service", null, []],
   ["WMS", "Web Mapping Service", "WMS", ["wms"]],
   ["WMTS", "Web Map Tile Service", null, []],

ckan/lib/cli.py

@@ -1889,7 +1889,7 @@ def less(self):
             f.close()
             self.compile_less(root, less_bin, color)
         f = open(custom_less, 'w')
-        f.write('// This file is needed in order for ./bin/less to compile in less 1.3.1+\n')
+        f.write('// This file is needed in order for `gulp build` to compile in less 1.3.1+\n')
         f.close()
         self.compile_less(root, less_bin, 'main')
 

ckan/tests/test_coding_standards.py

@@ -667,6 +667,10 @@ def find_unprefixed_string_literals(filename):
     u"ckanext/example_theme_docs/v21_custom_jquery_plugin/plugin.py",
     u"ckanext/imageview/plugin.py",
     u"ckanext/imageview/tests/test_view.py",
+    u'ckanext/audioview/plugin.py',
+    u'ckanext/audioview/tests/test_view.py',
+    u'ckanext/videoview/plugin.py',
+    u'ckanext/videoview/tests/test_view.py',
     u"ckanext/multilingual/plugin.py",
     u"ckanext/multilingual/tests/test_multilingual_plugin.py",
     u"ckanext/reclineview/plugin.py",

ckanext/audioview/plugin.py

@@ -0,0 +1,41 @@
+# encoding: utf-8
+
+from six import text_type
+import ckan.plugins as p
+
+ignore_empty = p.toolkit.get_validator('ignore_empty')
+
+DEFAULT_AUDIO_FORMATS = 'wav ogg mp3'
+
+
+class AudioView(p.SingletonPlugin):
+    '''This plugin makes views of audio resources, using an <audio> tag'''
+
+    p.implements(p.IConfigurer, inherit=True)
+    p.implements(p.IResourceView, inherit=True)
+
+    def update_config(self, config):
+        p.toolkit.add_template_directory(config, 'theme/templates')
+        self.formats = config.get(
+            'ckan.preview.audio_formats',
+            DEFAULT_AUDIO_FORMATS).split()
+
+    def info(self):
+        return {'name': 'audio_view',
+                'title': p.toolkit._('Audio'),
+                'icon': 'file-audio-o',
+                'schema': {'audio_url': [ignore_empty, text_type]},
+                'iframed': False,
+                'always_available': True,
+                'default_title': p.toolkit._('Audio'),
+                }
+
+    def can_view(self, data_dict):
+        return (data_dict['resource'].get('format', '').lower()
+                in self.formats)
+
+    def view_template(self, context, data_dict):
+        return 'audio_view.html'
+
+    def form_template(self, context, data_dict):
+        return 'audio_form.html'

ckanext/audioview/tests/test_view.py

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+from ckan.lib.helpers import url_for
+from ckan.tests import factories
+
+import ckan.plugins as p
+
+import pytest
+
+
+@pytest.mark.ckan_config('ckan.views.default_views', '')
+@pytest.mark.ckan_config("ckan.plugins", "audio_view")
+@pytest.mark.usefixtures("clean_db", "with_plugins")
+def test_view_shown_on_resource_page_with_audio_url(app):
+
+    dataset = factories.Dataset()
+
+    resource = factories.Resource(package_id=dataset['id'],
+                                  format='wav')
+
+    resource_view = factories.ResourceView(
+        resource_id=resource['id'],
+        view_type='audio_view',
+        audio_url='http://example.wav')
+
+    url = url_for('resource.read',
+                  id=dataset['name'], resource_id=resource['id'])
+
+    response = app.get(url)
+
+    assert resource_view['audio_url'] in response

ckanext/audioview/theme/templates/audio_form.html

@@ -0,0 +1,3 @@
+{% import 'macros/form.html' as form %}
+
+{{ form.input('audio_url', id='field-audio_url', label=_('Audio url'), placeholder=_('eg. http://example.com/audio.mp3  (if blank uses resource url)'), value=data.audio_url, error=errors.audio_url, classes=['control-full', 'control-large']) }}

ckanext/audioview/theme/templates/audio_view.html

@@ -0,0 +1,9 @@
+{% set res_url = resource_view.get('audio_url') or resource.get('url') %}
+
+<audio style="margin:auto; max-height:100%; display:block; min-width: 100%;"
+controls src="{{ resource_view.audio_url or resource.get('url') }}">
+  {% trans %}
+    Your browser does not support the <code>audio</code> element.
+    But don't worry, you can <a href="{{ res_url }}" alt="video url">download it</a>.
+  {% endtrans %}
+</audio>

ckanext/videoview/plugin.py

@@ -0,0 +1,42 @@
+# encoding: utf-8
+
+from six import text_type
+import ckan.plugins as p
+
+ignore_empty = p.toolkit.get_validator('ignore_empty')
+
+DEFAULT_VIDEO_FORMATS = 'mp4 ogg webm'
+
+
+class VideoView(p.SingletonPlugin):
+    '''This plugin makes views of video resources, using a <video> tag'''
+
+    p.implements(p.IConfigurer, inherit=True)
+    p.implements(p.IResourceView, inherit=True)
+
+    def update_config(self, config):
+        p.toolkit.add_template_directory(config, 'theme/templates')
+        self.formats = config.get(
+            'ckan.preview.video_formats',
+            DEFAULT_VIDEO_FORMATS).split()
+
+    def info(self):
+        return {'name': 'video_view',
+                'title': p.toolkit._('Video'),
+                'icon': 'file-video-o',
+                'schema': {'video_url': [ignore_empty, text_type],
+                           'poster_url': [ignore_empty, text_type]},
+                'iframed': False,
+                'always_available': True,
+                'default_title': p.toolkit._('Video'),
+                }
+
+    def can_view(self, data_dict):
+        return (data_dict['resource'].get('format', '').lower()
+                in self.formats)
+
+    def view_template(self, context, data_dict):
+        return 'video_view.html'
+
+    def form_template(self, context, data_dict):
+        return 'video_form.html'

ckanext/videoview/tests/test_view.py

@@ -0,0 +1,30 @@
+# encoding: utf-8
+
+from ckan.lib.helpers import url_for
+from ckan.tests import helpers, factories
+
+import ckan.plugins as p
+import pytest
+
+
+@pytest.mark.ckan_config('ckan.views.default_views', '')
+@pytest.mark.ckan_config("ckan.plugins", "video_view")
+@pytest.mark.usefixtures("clean_db", "with_plugins")
+def test_view_shown_on_resource_page_with_video_url(app):
+
+    dataset = factories.Dataset()
+
+    resource = factories.Resource(package_id=dataset['id'],
+                                  format='mp4')
+
+    resource_view = factories.ResourceView(
+        resource_id=resource['id'],
+        view_type='video_view',
+        video_url='https://example/video.mp4')
+
+    url = url_for('resource.read',
+                  id=dataset['name'], resource_id=resource['id'])
+
+    response = app.get(url)
+
+    assert resource_view['video_url'] in response

ckanext/videoview/theme/templates/video_form.html

@@ -0,0 +1,4 @@
+{% import 'macros/form.html' as form %}
+
+{{ form.input('video_url', id='field-video_url', label=_('Video url'), placeholder=_('eg. http://example.com/video.mpeg  (if blank uses resource url)'), value=data.video_url, error=errors.video_url, classes=['control-full', 'control-large']) }}
+{{ form.input('poster_url', id='field-poster_url', label=_('Poster url'), placeholder=_('eg. http://example.com/poster.jpg'), value=data.poster_url, error=errors.poster_url, classes=['control-full', 'control-large']) }}

ckanext/videoview/theme/templates/video_view.html

@@ -0,0 +1,12 @@
+{% set res_url = resource_view.get('video_url') or resource.get('url') %}
+
+<video style="margin:auto; max-height:100%; width: 250px; min-width: 100%; display:block"
+src="{{ resource_view.video_url or resource.get('url') }}"
+type="{{ resource.get('format') }}" controls
+poster="{{ resource_view.poster_url }}">
+  {% trans %}
+    Sorry, your browser doesn't support embedded videos,
+    but don't worry, you can <a href="{{ res_url }}" alt="video url">download it</a>
+    and watch it with your favorite video player!
+  {% endtrans %}
+</video>

doc/contributing/frontend/index.rst

@@ -25,7 +25,7 @@ Instructions for installing |nodejs| can be found on the |nodejs| `website
 On Ubuntu, run the following to install |nodejs| official repository and the node
 package::
 
-    curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash -
+    curl -sL https://deb.nodesource.com/setup_13.x | sudo -E bash -
     sudo apt-get install -y nodejs
 
 .. note:: If you use the package on the default Ubuntu repositories (eg ``sudo apt-get install nodejs``),
@@ -34,22 +34,18 @@ package::
 
         ln -s /usr/bin/nodejs /usr/bin/node
 
-    Also npm (the |nodejs| package manager) needs to be installed separately::
-
-        sudo apt-get install npm
-
     For more information, refer to the |nodejs| `instructions
     <https://nodejs.org/en/download/package-manager/#debian-and-ubuntu-based-linux-distributions>`_.
 
-Less can then be installed via the node package manager (npm).
-We also use ``nodewatch`` to make our Less compiler a watcher
+Dependencies can then be installed via the node package manager (npm).
+We use ``gulp`` to make our Less compiler a watcher
 style script.
 
 ``cd`` into the CKAN source folder (eg |virtualenv|/src/ckan ) and run:
 
 ::
 
-    $ npm install less@3.7.1 nodewatch
+    $ npm install
 
 
 You may need to use ``sudo`` depending on your CKAN install type.
@@ -127,11 +123,13 @@ before beginning development by running:
 
 ::
 
-    $ ./bin/less
+    $ npm run watch
 
 This will watch for changes to all of the less files and automatically
-rebuild the CSS for you. To quit the script press ``ctrl-c``. There is also
-``--production`` flag for compiling the production ``main.css``.
+rebuild the CSS for you. To quit the script press ``ctrl-c``. If you
+need sourcemaps for debugging, set `DEBUG` environment variable. I.e::
+
+  $ DEBUG=1 npm run watch
 
 There are many Less files which attempt to group the styles in useful
 groups. The main two are:
@@ -146,7 +144,7 @@ ckan.less:
 
 .. Note::
     Whenever a CSS change effects ``main.less`` it's important than after
-    the merge into master that a ``$ ./bin/less --production`` should be
+    the merge into master that a ``$ npm run build`` should be
     run and commited.
 
 There is a basic pattern primer available at:

doc/contributing/release-process.rst

@@ -96,7 +96,7 @@ Turn this file into a github issue with a checklist using this command::
    cherry-picked from master, the less compiling command needs to be run on
    the release branch. This will update the ``main.css`` file::
 
-        ./bin/less --production
+        npm run build
         git commit -am "Rebuild CSS"
         git push