Updating from main repo

.travis.yml

@@ -1,14 +1,14 @@
 language: python
 python:
   - "2.7"
-  - "3.3"
   - "3.4"
   - "3.5"
   - "3.6"
+  - "3.7"
   - "pypy"
 matrix:
   allow_failures:
-    - python: "3.3"
+    - python: "3.7"
     - python: "pypy"
 # command to install dependencies
 install:
@@ -26,3 +26,4 @@ notifications:
     - iemejia@gmail.com
     - kidsshk3@gmail.com
     - rbrito@ime.usp.br
+    - baltazar.bz@gmail.com

edx_dl/_version.py

@@ -1 +1 @@
-__version__ = '0.1.9'
+__version__ = '0.1.10'

edx_dl/edx_dl.py

@@ -702,6 +702,7 @@ def download_url(url, filename, headers, args):
         download_youtube_url(url, filename, headers, args)
     else:
         import ssl
+        import requests
         # FIXME: Ugly hack for coping with broken SSL sites:
         # https://www.cs.duke.edu/~angl/papers/imc10-cloudcmp.pdf
         #
@@ -715,7 +716,13 @@ def download_url(url, filename, headers, args):
         # order) is due to different behaviors in different Python versions
         # (e.g., 2.7 vs. 3.4).
         try:
-            urlretrieve(url, filename)
+            # mitxpro fix for downloading compressed files
+            if 'zip' in url and 'mitxpro' in url:
+                urlretrieve(url, filename)
+            else:
+                r = requests.get(url, headers=headers)
+                with open(filename, 'wb') as fp:
+                    fp.write(r.content)
         except Exception as e:
             logging.warn('Got SSL/Connection error: %s', e)
             if not args.ignore_errors:

edx_dl/parsing.py

@@ -14,8 +14,8 @@
 from .common import Course, Section, SubSection, Unit, Video
 
 
-# Force use of bs4 with html5lib
-BeautifulSoup = lambda page: BeautifulSoup_(page, 'html5lib')
+# Force use of bs4 with html.parser
+BeautifulSoup = lambda page: BeautifulSoup_(page, 'html.parser')
 
 
 def edx_json2srt(o):
@@ -388,7 +388,7 @@ def _make_subsections(section_soup):
             # FIXME correct extraction of subsection.name (unicode)
             subsections = [SubSection(position=i,
                                       url=s.a['href'],
-                                      name=s.a.div.span.string.strip())
+                                      name=s.a.div.div.string.strip())
                            for i, s in enumerate(subsections_soup, 1)]
 
             return subsections
@@ -412,7 +412,10 @@ def get_page_extractor(url):
     """
     factory method for page extractors
     """
-    if url.startswith('https://courses.edx.org'):
+    if (
+        url.startswith('https://courses.edx.org') or
+        url.startswith('https://mitxpro.mit.edu')
+    ):
         return NewEdXPageExtractor()
     elif (
         url.startswith('https://edge.edx.org') or

requirements.txt

@@ -1,4 +1,5 @@
-beautifulsoup4>=4.1.3
-html5lib>=1.0b2
-six>=1.5.0
-youtube_dl>=2015.05.20
+beautifulsoup4>=4.6.0
+html5lib>=1.0.1
+six>=1.11.0
+youtube_dl>=2018.06.18
+requests>=2.18.4

setup.py

@@ -72,12 +72,12 @@ def read_file(filename, alt=None):
     'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)',
     'Operating System :: OS Independent',
     'Programming Language :: Python :: 2',
-    'Programming Language :: Python :: 2.6',
     'Programming Language :: Python :: 2.7',
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.3',
     'Programming Language :: Python :: 3.4',
     'Programming Language :: Python :: 3.5',
+    'Programming Language :: Python :: 3.6',
+    'Programming Language :: Python :: 3.7',
     'Programming Language :: Python :: Implementation :: CPython',
     'Programming Language :: Python :: Implementation :: PyPy',
     'Programming Language :: Python',
@@ -87,8 +87,8 @@ def read_file(filename, alt=None):
 setup(
     name='edx-dl',
     version=__version__,
-    maintainer='Ismaël Mejía, Rogério Theodoro de Brito',
-    maintainer_email='iemejia@gmail.com, rbrito@ime.usp.br',
+    maintainer='Ismaël Mejía, Rogério Theodoro de Brito, Yuri Bochkarev',
+    maintainer_email='iemejia@gmail.com, rbrito@ime.usp.br, baltazar.bz@gmail.com',
 
     license='LGPL',
     url='https://github.com/coursera-dl/edx-dl',