Merge pull request #245 from ivanistheone/handle_more_errors

Organize testcontent dir + fix for description length issue #235

.gitignore

@@ -113,7 +113,8 @@ restore/
 node_modules/
 
 sushi_chefs/
-tests/testcontent/
+tests/testcontent/downloaded/
+tests/testcontent/generated/
 .webcache
 .ricecookerfilecache
 

Makefile

@@ -46,15 +46,17 @@ clean-test: ## remove test and coverage artifacts
 	rm -fr .tox/
 	rm -f .coverage
 	rm -fr htmlcov/
+	rm -rf tests/testcontent/downloaded/*
+	rm -rf tests/testcontent/generated/*
 
 lint: ## check style with flake8
 	flake8 ricecooker tests
 
-test: ## run tests quickly with the default Python
+test: clean-test ## run tests quickly with the default Python
 	pytest
 
 
-test-all: ## run tests on every Python version with tox
+test-all: clean-test ## run tests on every Python version with tox
 	tox
 
 coverage: ## check code coverage quickly with the default Python

ricecooker/classes/nodes.py

@@ -100,6 +100,7 @@ def generate_thumbnail(self):
     def has_thumbnail(self):
         from .files import ThumbnailFile
         return any(f for f in self.files if isinstance(f, ThumbnailFile))
+        # TODO deep check: f.process_file() and check f.filename is not None
 
     def set_thumbnail(self, thumbnail):
         """ set_thumbnail: Set node's thumbnail

ricecooker/managers/tree.py

@@ -188,6 +188,7 @@ def add_channel(self):
             Returns: link to uploadedchannel
         """
         config.LOGGER.info("   Creating channel {0}".format(self.channel.title))
+        self.channel.truncate_fields()
         payload = {
             "channel_data":self.channel.to_dict(),
         }

tests/conftest.py

@@ -1,4 +1,5 @@
 import copy
+import glob
 import os
 import pytest
 import requests
@@ -14,6 +15,18 @@
 
 
 
+# GLOBAL TEST SETUP/TEARDOWN UTILS
+################################################################################
+
+def pytest_sessionfinish(session, exitstatus):
+    """
+    Cleanup testcontent/generated/ directory after each test run is finished.
+    """
+    generated_path = os.path.join("tests", "testcontent", "generated")
+    for path in glob.glob(generated_path + os.path.sep + '*'):
+        os.remove(path)
+
+
 # CHANNEL FIXTURES
 ################################################################################
 
@@ -251,7 +264,7 @@ def contentnode_no_source_id(title):
 @pytest.fixture
 def video_file():    # uses same file as test_videos.low_res_video fixture
     source_url = "https://archive.org/download/vd_is_for_everybody/vd_is_for_everybody_512kb.mp4"
-    local_path = os.path.join("tests", "testcontent", "low_res_video.mp4")
+    local_path = os.path.join("tests", "testcontent", "downloaded", "low_res_video.mp4")
     download_fixture_file(source_url, local_path)
     assert os.path.exists(local_path)
     return VideoFile(local_path)
@@ -262,13 +275,14 @@ def video_filename():
 
 @pytest.fixture
 def subtitle_file():
-    if not os.path.exists("tests/testcontent/testsubtitles.vtt"):
-        with open("tests/testcontent/testsubtitles.vtt", 'wb') as subtitlefile:
+    local_path = os.path.join("tests", "testcontent", "generated", "testsubtitles.vtt")
+    if not os.path.exists(local_path):
+        with open(local_path, 'wb') as subtitlefile:
             subtitlefile.write(b'WEBVTT\n')
             subtitlefile.write(b'\n')
             subtitlefile.write(b'00:01.000 --> 00:04.250\n')
             subtitlefile.write(b'Testing subtitles\n')
-    return SubtitleFile("tests/testcontent/testsubtitles.vtt", language='en')
+    return SubtitleFile(local_path, language='en')
 
 @pytest.fixture
 def subtitle_filename():
@@ -304,7 +318,7 @@ def video_invalid_files(video_data, document_file):
 
 @pytest.fixture
 def invalid_video_file():
-    local_path = os.path.join("tests", "testcontent", "invalid_video.mp4")
+    local_path = os.path.join("tests", "testcontent", "generated", "invalid_video.mp4")
     if not os.path.exists(local_path):
         with open(local_path, 'wb') as f:
             f.write(b'this is an invalid video file')
@@ -318,7 +332,7 @@ def invalid_video_file():
 def audio_file():
     source_url = "https://ia800103.us.archive.org/9/items/cd_prince_prince/" \
                  "disc1/02.%20Prince%20-%201999%20%28Edit%29_sample.mp3"
-    local_path = os.path.join("tests", "testcontent", "testaudio.mp3")
+    local_path = os.path.join("tests", "testcontent", "downloaded", "testaudio.mp3")
     download_fixture_file(source_url, local_path)
     assert os.path.exists(local_path)
     return AudioFile(local_path)
@@ -357,7 +371,7 @@ def audio_invalid_files(audio_data, document_file):
 
 @pytest.fixture
 def invalid_audio_file():
-    local_path = os.path.join("tests", "testcontent", "invalid_audio.mp3")
+    local_path = os.path.join("tests", "testcontent", "generated", "invalid_audio.mp3")
     if not os.path.exists(local_path):
         with open(local_path, 'wb') as f:
             f.write(b'invalid MP3')
@@ -369,7 +383,7 @@ def invalid_audio_file():
 @pytest.fixture
 def document_file():
     source_url = "https://ia802506.us.archive.org/8/items/generalmanual_000075878/generalmanual_000075878.pdf"
-    local_path = os.path.join("tests", "testcontent", "testdocument.pdf")
+    local_path = os.path.join("tests", "testcontent", "downloaded", "testdocument.pdf")
     download_fixture_file(source_url, local_path)
     assert os.path.exists(local_path)
     return DocumentFile(local_path)
@@ -408,11 +422,9 @@ def document_invalid_files(document_data, audio_file):
 
 @pytest.fixture
 def epub_file():
-    source_url = "https://archive.org/download/generalmanual_000075878/generalmanual_000075878.epub"
-    local_path = os.path.join("tests", "testcontent", "testdocument.epub")
-    download_fixture_file(source_url, local_path)
-    assert os.path.exists(local_path)
-    return EPubFile(local_path)
+    path = os.path.join("tests", "testcontent", "samples", "testdocument.epub")
+    assert os.path.exists(path)
+    return EPubFile(path)
 
 @pytest.fixture
 def epub_filename():
@@ -421,15 +433,15 @@ def epub_filename():
 
 @pytest.fixture
 def invalid_document_file():
-    local_path = os.path.join("tests", "testcontent", "invalid_document.pdf")
+    local_path = os.path.join("tests", "testcontent", "generated", "invalid_document.pdf")
     if not os.path.exists(local_path):
         with open(local_path, 'wb') as f:
             f.write(b'invalid PDF')
     return DocumentFile(local_path)
 
 @pytest.fixture
 def invalid_epub_file():
-    local_path = os.path.join("tests", "testcontent", "invalid_document.epub")
+    local_path = os.path.join("tests", "testcontent", "generated", "invalid_document.epub")
     if not os.path.exists(local_path):
         with open(local_path, 'wb') as f:
             f.write(b'invalid ePub')
@@ -444,7 +456,7 @@ def html_file():
     source_url = "https://web.archive.org/web/20191122220303/" \
                  "https://studio.learningequality.org/content/storage/" \
                  "e/d/ed494d6547b603b8ff22095cf5f5b624.zip"
-    local_path = os.path.join("tests", "testcontent", "testhtml.zip")
+    local_path = os.path.join("tests", "testcontent", "downloaded", "testhtml.zip")
     download_fixture_file(source_url, local_path)
     assert os.path.exists(local_path)
     return HTMLZipFile(local_path)
@@ -454,22 +466,6 @@ def html_filename():
     return 'ed494d6547b603b8ff22095cf5f5b624.zip'
 
 
-@pytest.fixture
-def minimal_html_file():
-    if not os.path.exists("tests/testcontent/minimaltesthtml.zip"):
-        with zipfile.ZipFile("tests/testcontent/minimaltesthtml.zip", 'w', zipfile.ZIP_DEFLATED) as archive:
-            info = zipfile.ZipInfo('index.html', date_time=(2013, 3, 14, 1, 59, 26))
-            info.comment = "test file".encode()
-            info.compress_type = zipfile.ZIP_STORED
-            info.create_system = 0
-            archive.writestr(info, '<div></div>')
-    return HTMLZipFile("tests/testcontent/minimaltesthtml.zip")
-
-@pytest.fixture
-def minimal_html_filename():
-    return 'b3f06ea298da23eee1d231795655e01b.zip'
-
-
 @pytest.fixture
 def html_data(contentnode_base_data, html_file, channel_domain_namespace, channel_node_id):
     html_data = copy.deepcopy(contentnode_base_data)
@@ -504,10 +500,11 @@ def html_invalid_files(html_data, document_file):
 
 @pytest.fixture
 def html_invalid_file():
-    if not os.path.exists("tests/testcontent/testinvalidhtml.zip"):
-        with zipfile.ZipFile("tests/testcontent/testinvalidhtml.zip", 'w', zipfile.ZIP_DEFLATED) as archive:
+    local_path = os.path.join("tests", "testcontent", "generated", "testinvalidhtml.zip")
+    if not os.path.exists(local_path):
+        with zipfile.ZipFile(local_path, 'w', zipfile.ZIP_DEFLATED) as archive:
             archive.writestr("notindex.html", '<div></div>')
-    return HTMLZipFile("tests/testcontent/testinvalidhtml.zip")
+    return HTMLZipFile(local_path)
 
 @pytest.fixture
 def html_invalid_zip(html_data, html_invalid_file):
@@ -568,19 +565,21 @@ def exercise_invalid_question(exercise):
 
 @pytest.fixture
 def thumbnail_file():
-    assert os.path.exists("tests/testcontent/sample_thumbnail.png")
-    return ThumbnailFile("tests/testcontent/sample_thumbnail.png")
+    local_path = os.path.join("tests", "testcontent", "samples", "thumbnail.png")
+    assert os.path.exists(local_path)
+    return ThumbnailFile(local_path)
 
 @pytest.fixture
 def thumbnail_filename():
     return 'eb79354ddd5774bb3436f9a19c282bff.png'
 
 @pytest.fixture
 def fake_thumbnail_file():
-    if not os.path.exists("tests/testcontent/testimage.png"):
-        with open("tests/testcontent/testimage.png", 'wb') as imgfile:
+    local_path = os.path.join("tests", "testcontent", "generated", "invalidimage.png")
+    if not os.path.exists(local_path):
+        with open(local_path, 'wb') as imgfile:
             imgfile.write(b'not_a_valid_PNG')
-    return ThumbnailFile("tests/testcontent/testimage.png")
+    return ThumbnailFile(local_path)
 
 
 
@@ -589,7 +588,7 @@ def fake_thumbnail_file():
 
 @pytest.fixture
 def exercise_image_file():
-    return _ExerciseImageFile('tests/testcontent/no-wifi.png')
+    return _ExerciseImageFile('tests/testcontent/exercises/no-wifi.png')
 
 @pytest.fixture
 def exercise_image_filename():
@@ -598,7 +597,7 @@ def exercise_image_filename():
 
 @pytest.fixture
 def exercise_base64_image_file():
-    with open('tests/testcontent/test_image_base64.data') as datafile:
+    with open('tests/testcontent/exercises/test_image_base64.data') as datafile:
         base64_data = datafile.read()
         return _ExerciseBase64ImageFile(base64_data)
 
@@ -609,7 +608,7 @@ def exercise_base64_image_filename():
 
 @pytest.fixture
 def exercise_graphie_file():
-    return _ExerciseGraphieFile('tests/testcontent/eb3f3bf7c317408ee90995b5bcf4f3a59606aedd')
+    return _ExerciseGraphieFile('tests/testcontent/exercises/eb3f3bf7c317408ee90995b5bcf4f3a59606aedd')
 
 @pytest.fixture
 def exercise_graphie_replacement_str():
@@ -629,7 +628,7 @@ def exercise_graphie_filename():
 def slideshow_files():
     fake_files = []
     for i in range(0,10):
-        filename = 'tests/testcontent/slide' + str(i) + '.jpg'
+        filename = 'tests/testcontent/generated/slide' + str(i) + '.jpg'
         if not os.path.exists(filename):
             with open(filename, 'w') as f:
                 f.write('jpgdatawouldgohere' + str(i))
@@ -676,3 +675,5 @@ def download_fixture_file(source_url, local_path):
             f.write(chunk)
         f.flush()
         f.close()
+
+

tests/test_exercises.py

@@ -231,7 +231,7 @@ def test_inputquestion_validate():
 @pytest.fixture
 def graphie_strings_and_rawpath():
     """
-    Return patterns that should match the 
+    Return patterns that should match the
     WEB_GRAPHIE_URL_REGEX = r'web\+graphie:(?P<rawpath>[^\)]+)'
     """
     test_data = {
@@ -291,7 +291,7 @@ def image_texts_fixtures():
     """
     WEB_GRAPHIE_PREFIX = 'web+graphie:${☣ CONTENTSTORAGE}/'
     WEB_PREFIX = '${☣ CONTENTSTORAGE}/'
-    
+
     test_data = [
         {
             'text': 'web+graphie://ka-perseus-graphie.s3.amazonaws.com/eb3f3bf7c317408ee90995b5bcf4f3a59606aedd',
@@ -314,7 +314,7 @@ def image_texts_fixtures():
             'hash': '599aa896313be22dea6c0257772a464e'
          },
          {  # slightly modified version of the above
-            'text': os.path.relpath(os.path.join(TESTCONTENT_DIR, 'no-wifi.png')),
+            'text': os.path.relpath(os.path.join(TESTCONTENT_DIR, 'exercises', 'no-wifi.png')),
             'replacement_str': WEB_PREFIX + '599aa896313be22dea6c0257772a464e.png',
             'hash': '599aa896313be22dea6c0257772a464e'
          },
@@ -334,7 +334,7 @@ def test_base_question_set_image(image_texts_fixtures):
         text = datum['text']
         replacement_str = datum['replacement_str']
 
-        
+
         # SIT ##################################################################
         testq = BaseQuestion(id='someid', question='somequestion', question_type='input', raw_data={})
         new_text, images = testq.set_image(text)
@@ -404,7 +404,7 @@ def test_perseus__recursive_url_find(persues_question_json_fixtures):
     # checks
     new_url = test_data['question']['widgets']['interactive-graph 1']['options']['backgroundImage']['url']
     assert '☣ CONTENTSTORAGE' in new_url, 'url replacement not done'
-    assert hash in new_url, 'wrong url replacement'    
+    assert hash in new_url, 'wrong url replacement'
     assert len(image_files) == 1
     image_file = image_files[0]
     filename = image_file.get_filename()
@@ -423,7 +423,7 @@ def persues_contentimages_field_fixtures():
     Returns a list of data needed to test the `process_image_field` method:
       - `field`: input sample data
       - `new_content`: what the content field should get rewritten to
-      - `image_hashes`: content hash of image files that should get downloaded 
+      - `image_hashes`: content hash of image files that should get downloaded
     """
     test_data = [
       # Known good test cases from KA English exercise
@@ -440,7 +440,7 @@ def persues_contentimages_field_fixtures():
                     'widgets': {}
                    },
           'new_content': 'The function $f$\n![graph](web+graphie:${☣ CONTENTSTORAGE}/d8daa074ec7d09ce3819d6259b3e4670701d2540)',
-          'image_hashes': ['db98ca9d35b2fb97cde378a1fabddd26'],         
+          'image_hashes': ['db98ca9d35b2fb97cde378a1fabddd26'],
       },
       #
       # Same as above two but with missing images
@@ -457,7 +457,7 @@ def persues_contentimages_field_fixtures():
                     'widgets': {}
                    },
           'new_content': 'The function $f$\n![graph](web+graphie:${☣ CONTENTSTORAGE}/d8daa074ec7d09ce3819d6259b3e4670701d2540)',
-          'image_hashes': ['db98ca9d35b2fb97cde378a1fabddd26'],         
+          'image_hashes': ['db98ca9d35b2fb97cde378a1fabddd26'],
       },
     ]
     return test_data
@@ -513,7 +513,7 @@ def persues_question_json_fixtures():
     Load entire perseus questions
     """
     test_data = []
-    with open(os.path.join(TESTCONTENT_DIR, 'perseus_question_x43bbec76d5f14f88_en.json')) as inf:
+    with open(os.path.join(TESTCONTENT_DIR, 'exercises', 'perseus_question_x43bbec76d5f14f88_en.json')) as inf:
         # ENGLISH JSON = KNOWN GOOD
         item_data_en = json.load(inf)
         datum = {
@@ -523,8 +523,8 @@ def persues_question_json_fixtures():
         test_data.append(datum)
     # Missing images in the KA BULGARIAN channel BUG
     # see https://github.com/learningequality/ricecooker/issues/178
-    with open(os.path.join(TESTCONTENT_DIR, 'perseus_question_x43bbec76d5f14f88_bg.json')) as inf:
-        
+    with open(os.path.join(TESTCONTENT_DIR, 'exercises', 'perseus_question_x43bbec76d5f14f88_bg.json')) as inf:
+
         item_data_bg = json.load(inf)
         datum = {
             'item': item_data_bg,
@@ -540,7 +540,7 @@ def test_perseus_process_question(persues_question_json_fixtures):
     Process a persues question and check that it finds all images, and returns
     correcrt image files -- i.e not more, not less.
     """
-    
+
     for datum in persues_question_json_fixtures:
 
         # setup