diff --git a/.jscsrc b/.jscsrc index e20d9089..19e68fd3 100644 --- a/.jscsrc +++ b/.jscsrc @@ -1,5 +1,8 @@ { "preset": "google", "validateQuoteMarks": null, - "requireCamelCaseOrUpperCaseIdentifiers": "ignoreProperties" + "requireCamelCaseOrUpperCaseIdentifiers": "ignoreProperties", + "fileExtensions": [".js", ".jsx"], + "esprima": "esprima-fb", + "disallowSpacesInAnonymousFunctionExpression": null } diff --git a/README.rst b/README.rst index 188e425e..e63e6ac5 100644 --- a/README.rst +++ b/README.rst @@ -103,7 +103,12 @@ requirements file, just run ``tox``. The project also contains JavaScript tests which can be run using [Karma](karma-runner.github.io). ``tox`` will run the JavaScript tests after the Python tests. You can run only the JavaScript tests using -``docker-compose run web tox -e js``. +``docker-compose run web tox -e js``, or do continuous JavaScript +testing with ``docker-compose -f docker-karma.yml up`` and connecting +to port 9876 on your docker host. + +In addition to local testing, all commits and pull requests are tested +on travis-ci.org. Continuous Testing ~~~~~~~~~~~~~~~~~~ diff --git a/RELEASE.rst b/RELEASE.rst index 67a38d51..bf7ac3eb 100644 --- a/RELEASE.rst +++ b/RELEASE.rst @@ -1,6 +1,57 @@ Release Notes ------------- +Version 0.4.0 +============= + +- Added view to serve static assets and modified REST API +- Added fix and test for handling deleted Elasticsearch index. +- Refactored manage_taxonomies.jsx and related tests +- Sped up test discovery by removing node_modules from search +- Added learning resource types to manage taxonomies UI +- Added learning_resource_types API and learning_resource_types field for + vocabularies +- Fixed bug with file path length in static assets +- Added learning resource UI to edit description and terms +- Upgraded several packages + - Bootstrap + - uwsgi + - static3 + - elasticsearch + - django-bootstrap + - django-storages-redux +- Added terms to the readonly list +- Allowed blank descriptions for LearningResource model +- Implemented Enter key to add taxonomy term and added test case to + fix coverage +- Updated Django to 1.8.3 +- Correct LORE production URL in Apiary doc +- Added checkbox styling to vocabulary/term facets +- Fixed error message on unsupported terms in learning resource +- Fixed facet checkboxes not showing in production +- Fixed course/run highlight bug +- Default checked radio button for Manage Taxonomies -> Add Vocabulary +- Fixed vertical alignment of taxonomy tabs +- Fixed error message for duplicate vocabulary +- Added docker container for javascript testing +- Added checkboxes and ability to toggle facets +- Added html coverage report for javascript +- Added shim configuration to karma test runner +- Implemented learning_resources API +- Members REST API docs +- Linked video transcripts to learning resources. +- Parse static assets from LearningResource +- Removed unused patterns to limit memory use +- fix css to make list vertical align +- Installed JSXHint and configured JSCS to work with JSX files +- Included JSX files in coverage results +- Allow only usernames and not emails in the Members add input +- Added test case, tested menulay all scenarios +- Moved coverage CLI script to utils directory +- Fixed buttons alignment problem in members panel. +- Fixed error message behavior for manage taxonomies tab +- Added ability to filter vocabularies by learning resource type + Version 0.3.0 ============= @@ -28,13 +79,13 @@ Version 0.2.0 from the right side of the page. - Glyphs for learning resources types are displayed in the left side panel for facets. -- LORE's RESTful web service documentation is available. +- LORE's RESTful web service documentation is available. (http://docs.lore.apiary.io) - Authorizations are in place for taxonomy endpoints in LORE's web service. - Relationships between learning resources and static assets are captured. -- Roles app has additional features. +- Roles app has additional features. Other Changes ************* @@ -69,4 +120,4 @@ Version 0.1.0 - Protected export view - Added faceted filtering - Added new manage repo users permission -- Fixed repository listing page to only show results for a single repo. +- Fixed repository listing page to only show results for a single repo. diff --git a/apiary.apib b/apiary.apib index 2f9ab20a..57c8ecf2 100644 --- a/apiary.apib +++ b/apiary.apib @@ -1,5 +1,5 @@ FORMAT: 1A -HOST: http://lore.mit.edu/ +HOST: http://lore.odl.mit.edu/ # LORE @@ -17,11 +17,11 @@ URL Structure is `https://{domain}/api/v1/{resource}/{resource_id}/` |`{resource}` | The resource endpoint for a specific set of items in the system. | |`{resource_id}` |The `resource_id` sets the unique identifier (name or numerical) for a specific item to reference. | -*Example: `http://lore.mit.edu/api/v1/repositories/` will return a JSON representation of all repositories.* +*Example: `http://lore.odl.mit.edu/api/v1/repositories/` will return a JSON representation of all repositories.* -*Example: `http://lore.mit.edu/api/v1/repositories/physics-1/` will return a JSON representation of the repository with name "physics-1".* +*Example: `http://lore.odl.mit.edu/api/v1/repositories/physics-1/` will return a JSON representation of the repository with name "physics-1".* -*Example: `http://lore.mit.edu/api/v1/repositories/physics-1/learningresources/1007/` will return a JSON representation +*Example: `http://lore.odl.mit.edu/api/v1/repositories/physics-1/learningresources/1007/` will return a JSON representation of the learning resource with ID=1007.* @@ -61,7 +61,7 @@ The API uses basic or session-based authentication to authenticate users using t Requests that return multiple items accept a `?page` parameter. -Example: `http://lore.mit.edu/api/v1/repositories/?page=2` +Example: `http://lore.odl.mit.edu/api/v1/repositories/?page=2` Page numbering is 1-based. Omitting the `?page` parameter will return the first page. There are up to 20 results per page. @@ -136,37 +136,167 @@ Create a repository by providing its JSON representation. ## Group Learningresources -## Learningresource Collection [/repositories/{repo_slug}/learningresources/{?learning_resource_type%5b%5d,course_id,xa_nr_views,page}] +## Learningresource Collection [/repositories/{repo_slug}/learning_resources/{?type_name}] + Parameters + repo_slug: `physics-1` (string, required) - slug for the repository - + learning_resource_type: `chapter` (string, optional) - types of learning resources - + course_id: (string, optional) + + type_name: `chapter` (string, optional) - type of learning resource -### List learning resources [GET] +### List Learningresources [GET] + Response 200 (application/json) + Body - [ - { - "tbd": "tbd", - } - ] + { + "count": 139, + "next": "http://lore.odl.mit.edu/api/v1/repositories/repo/learning_resources/?page=2", + "previous": null, + "results": [ + { + "id": 8, + "learning_resource_type": "vertical", + "static_assets": [], + "title": "Getting Started", + "description": "description", + "content_xml": " @@ -58,7 +62,3 @@ celery: - db - elastic - redis -elastic: - image: elasticsearch - ports: - - "9200" diff --git a/docker-karma.yml b/docker-karma.yml new file mode 100644 index 00000000..d19fa054 --- /dev/null +++ b/docker-karma.yml @@ -0,0 +1,7 @@ +karma: + image: lore_web + ports: + - "9876:9876" + command: tox -e js -- --no-single --no-single-run --auto-watch + volumes: + - .:/src diff --git a/docs/jsdocs.conf.json b/docs/jsdocs.conf.json new file mode 100644 index 00000000..b828e006 --- /dev/null +++ b/docs/jsdocs.conf.json @@ -0,0 +1,3 @@ +{ + "plugins": ["plugins/markdown"] +} diff --git a/importer/api/__init__.py b/importer/api/__init__.py index 7d4c30a6..560f8db7 100644 --- a/importer/api/__init__.py +++ b/importer/api/__init__.py @@ -8,7 +8,7 @@ import logging from tempfile import mkdtemp from os.path import join, exists, isdir -from os import listdir, walk, sep +from os import listdir from archive import Archive, ArchiveException from django.core.files import File @@ -17,10 +17,10 @@ from xbundle import XBundle, DESCRIPTOR_TAGS from learningresources.api import ( - create_course, - create_resource, - create_static_asset + create_course, create_resource, import_static_assets, + create_static_asset, get_video_sub, ) +from learningresources.models import StaticAsset, course_asset_basepath log = logging.getLogger(__name__) @@ -42,8 +42,6 @@ def import_course_from_file(filename, repo_id, user_id): None Raises: ValueError: Unable to extract or read archive contents. - - """ tempdir = mkdtemp() @@ -95,14 +93,12 @@ def import_course_from_path(path, repo_id, user_id): """ bundle = XBundle() bundle.import_from_directory(path) - course = import_course(bundle, repo_id, user_id) static_dir = join(path, 'static') - if isdir(static_dir): - import_static_assets(static_dir, course) + course = import_course(bundle, repo_id, user_id, static_dir) return course -def import_course(bundle, repo_id, user_id): +def import_course(bundle, repo_id, user_id, static_dir): """ Import a course from an XBundle object. @@ -110,6 +106,7 @@ def import_course(bundle, repo_id, user_id): bundle (xbundle.XBundle): Course as xbundle XML repo_id (int): Primary key of repository course belongs to user_id (int): Primary key of Django user doing the import + static_dir (unicode): location of static files Returns: learningresources.models.Course """ @@ -121,6 +118,7 @@ def import_course(bundle, repo_id, user_id): run=src.attrib["semester"], user_id=user_id, ) + import_static_assets(course, static_dir) import_children(course, src, None) return course @@ -144,26 +142,16 @@ def import_children(course, element, parent): content_xml=etree.tostring(element), mpath=mpath, ) + if element.tag == "video": + subname = get_video_sub(element) + if subname != "": + assets = StaticAsset.objects.filter( + course__id=resource.course_id, + asset=course_asset_basepath(course, subname), + ) + for asset in assets: + resource.static_assets.add(asset) + for child in element.getchildren(): if child.tag in DESCRIPTOR_TAGS: import_children(course, child, resource) - - -def import_static_assets(path, course): - """ - Upload all assets and create model records of them for a given - course and path. - - Args: - path (unicode): course specific path to extracted OLX tree. - course (learningresources.models.Course): Course to add assets to. - Returns: - None - """ - for root, _, files in walk(path): - for name in files: - with open(join(root, name), 'r') as open_file: - django_file = File(open_file) - # Remove base path from file name - django_file.name = join(root, name).replace(path + sep, '', 1) - create_static_asset(course.id, django_file) diff --git a/importer/tests/test_import.py b/importer/tests/test_import.py index 2a4db618..16b12830 100644 --- a/importer/tests/test_import.py +++ b/importer/tests/test_import.py @@ -4,10 +4,12 @@ from __future__ import unicode_literals +from collections import namedtuple import os from shutil import rmtree from tempfile import mkstemp, mkdtemp import zipfile +import logging from django.core.files.storage import default_storage from django.core.files.base import ContentFile @@ -16,13 +18,17 @@ from importer.api import ( import_course_from_file, import_course_from_path, - import_static_assets + import_static_assets, ) from importer.tasks import import_file from learningresources.api import get_resources -from learningresources.models import Course, StaticAsset, static_asset_basepath +from learningresources.models import ( + Course, StaticAsset, static_asset_basepath, LearningResource, +) from learningresources.tests.base import LoreTestCase +log = logging.getLogger(__name__) + class TestImportToy(LoreTestCase): """ @@ -131,7 +137,7 @@ def test_import_course_from_path(): with mock.patch('importer.api.import_course') as mock_import: with mock.patch( 'importer.api.import_static_assets' - ) as mock_static: + ): with mock.patch('importer.api.XBundle') as mock_bundle: with mock.patch('importer.api.isdir') as mock_is_dir: mock_import.return_value = True @@ -140,10 +146,8 @@ def test_import_course_from_path(): test_path, test_repo_id, test_user_id ) mock_import.assert_called_with( - mock_bundle(), test_repo_id, test_user_id - ) - mock_static.assert_called_with( - os.path.join(test_path, 'static'), True + mock_bundle(), test_repo_id, + test_user_id, os.path.join(test_path, "static"), ) def test_import_static_assets(self): @@ -157,7 +161,7 @@ def test_import_static_assets(self): with open(os.path.join(temp_dir_path, basename), 'w') as temp: temp.write(file_contents) # All setup, now import - import_static_assets(temp_dir_path, self.course) + import_static_assets(self.course, temp_dir_path) assets = StaticAsset.objects.filter(course=self.course) self.assertEqual(assets.count(), 1) asset = assets[0] @@ -187,7 +191,7 @@ def test_import_static_recurse(self): temp.write(file_contents) # All setup, now import - import_static_assets(temp_dir_path, self.course) + import_static_assets(self.course, temp_dir_path) assets = StaticAsset.objects.filter(course=self.course) self.assertEqual(assets.count(), 1) asset = assets[0] @@ -217,10 +221,60 @@ def test_static_import_integration(self): assets = StaticAsset.objects.filter(course=course) for asset in assets: self.addCleanup(default_storage.delete, asset.asset) - self.assertEqual(assets.count(), 2) + self.assertEqual(assets.count(), 3) for asset in assets: base_path = static_asset_basepath(asset, '') self.assertIn( asset.asset.name.replace(base_path, ''), - ['test.txt', 'subdir/subtext.txt'] + [ + 'test.txt', 'subdir/subtext.txt', + 'subs_CCxmtcICYNc.srt.sjson' + ] ) + + def test_parse_static(self): + """ + Parse the static assets in the sample course + """ + def get_counts(): + """Returns counts of resources, videos, and assets.""" + counts = namedtuple("counts", "resources videos assets") + kwargs = {"course__course_number": "toy"} + resources = LearningResource.objects.filter(**kwargs).count() + assets = StaticAsset.objects.filter(**kwargs).count() + kwargs["learning_resource_type__name"] = "video" + videos = LearningResource.objects.filter(**kwargs).count() + return counts(resources, videos, assets) + + # There should be nothing. + counts = get_counts() + self.assertTrue(counts.resources == 0) + self.assertTrue(counts.videos == 0) + self.assertTrue(counts.assets == 0) + # Import the course. + import_course_from_file( + self.get_course_single_tarball(), + self.repo.id, self.user.id + ) + # There should be something. + counts = get_counts() + self.assertTrue(counts.resources == 5) + self.assertTrue(counts.videos == 2) + self.assertTrue(counts.assets == 3) + # Only one video in the course has subtitles. + self.assertTrue( + StaticAsset.objects.filter( + learningresource__course__id__isnull=False).count() == 1) + + # There should be a single static asset. + course = Course.objects.all().order_by("-id")[0] # latest course + videos = LearningResource.objects.filter( + learning_resource_type__name="video", + course__id=course.id + ) + self.assertTrue(videos.count() == 2) + num_assets = sum([ + video.static_assets.count() + for video in videos + ]) + self.assertTrue(num_assets == 1) diff --git a/karma.conf.js b/karma.conf.js index d6aeb0f0..6eef62b1 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -15,30 +15,55 @@ module.exports = function(config) { // list of files / patterns to load in the browser files: [ - 'ui/jstests/test-listing.js', + 'ui/static/ui/js/require_config.js', + 'ui/jstests/test-main.js', { - pattern: 'ui/static/bower/react/react.js', + pattern: 'ui/static/bower/**/*.js', included: false }, { - pattern: 'ui/static/ui/js/**/*.js*', + pattern: 'ui/static/bower/**/*.jsx', included: false }, { - pattern: 'ui/jstests/**/*.js*', + pattern: 'ui/static/ui/**/*.js', included: false - } + }, + { + pattern: 'ui/static/ui/**/*.jsx', + included: false + }, + { + pattern: 'ui/jstests/**/*.js', + included: false + }, + { + pattern: 'ui/jstests/**/*.jsx', + included: false + }, + { + pattern: 'node_modules/jquery-mockjax/src/jquery.mockjax.js', + included: false + }, + { + pattern: 'node_modules/stacktrace-js/stacktrace.js', + included: false + }, ], - // list of files to exclude - exclude: [], + // list of files to exclude from coverage and testing + exclude: [ + "ui/static/ui/js/listing.js", + "ui/static/ui/js/csrf.js" + ], // preprocess matching files before serving them to the browser // available preprocessors: https://npmjs.org/browse/keyword/karma-preprocessor preprocessors: { - '**/*.jsx': ['react'], - 'ui/static/ui/js/**/*.js*': ['coverage'], - 'ui/jstests/**/*.js*': ['coverage'] + 'ui/static/ui/**/*.jsx': ['react', 'coverage'], + 'ui/jstests/**/*.jsx': ['react', 'coverage'], + 'ui/static/ui/**/*.js': ['coverage'], + 'ui/jstests/**/*.js': ['coverage'] }, reactPreprocessor: { @@ -57,6 +82,10 @@ module.exports = function(config) { coverageReporter: { dir: 'coverage/', reporters: [ + { + type: 'lcov', + subdir: '.', + }, { type: 'lcovonly', subdir: '.', diff --git a/learningresources/api.py b/learningresources/api.py index ee852d2f..6b813337 100644 --- a/learningresources/api.py +++ b/learningresources/api.py @@ -6,8 +6,11 @@ from __future__ import unicode_literals import logging +from os import walk, sep +from os.path import join from django.contrib.auth.models import User +from django.core.files import File from django.db import transaction from guardian.shortcuts import get_objects_for_user, get_perms @@ -222,30 +225,71 @@ def get_resource(resource_id, user_id): return resource -def create_static_asset(course_id, file_handle): +def create_static_asset(course_id, handle): """ - Create a static asset for a given course. + Create a static asset. + Args: + course_id (int): learningresources.models.Course pk + handle (django.core.files.File): file handle + Returns: + learningresources.models.StaticAsset + """ + with transaction.atomic(): + return StaticAsset.objects.create(course_id=course_id, asset=handle) - Warning: - This takes and open file handle and does not close it. You - will need to handle the opening and closing the - ``file_handle`` outside of this method. - Raises: - ValueError: If a closed handle is passed in +def _subs_filename(subs_id, lang='en'): + """ + Generate proper filename for storage. + + Function copied from: + edx-platform/common/lib/xmodule/xmodule/video_module/transcripts_utils.py Args: - course_id (int): Primary key of the Course to add the asset to. - file_handle (django.core.files.File): - An open file handle to save to the model. + subs_id (str): Subs id string + lang (str): Locale language (optional) default: en Returns: - learningresources.models.StaticAsset + filename (str): Filename of subs file """ - course = Course.objects.get(id=course_id) - with transaction.atomic(): - static_asset, _ = StaticAsset.objects.get_or_create( - course=course, - asset=file_handle - ) - return static_asset + if lang in ('en', "", None): + return u'subs_{0}.srt.sjson'.format(subs_id) + else: + return u'{0}_subs_{1}.srt.sjson'.format(lang, subs_id) + + +def get_video_sub(xml): + """ + Get subtitle IDs from