Merge pull request #84 from Erotemic/dev/doctest_jupyter

Dev/doctest jupyter

.circleci/config.yml

@@ -49,7 +49,7 @@ jobs:
       # ### INITIALIZE AND CACHE REQUIREMENTS ###
       - restore_cache:
           keys:
-          - v2-dependencies-{{ checksum "requirements/runtime.txt" }}-{{ checksum "requirements/tests.txt" }}
+          - v3-dependencies-{{ checksum "requirements/runtime.txt" }}-{{ checksum "requirements/tests.txt" }}-{{ checksum "requirements/jupyter.txt" }}
       - run:
           name: install dependencies
           command: |
@@ -63,14 +63,15 @@ jobs:
       - save_cache:
           paths:
             - ./venv
-          key: v2-dependencies-{{ checksum "requirements/runtime.txt" }}-{{ checksum "requirements/tests.txt" }}
+          key: v3-dependencies-{{ checksum "requirements/runtime.txt" }}-{{ checksum "requirements/tests.txt" }}-{{ checksum "requirements/jupyter.txt" }}
       # ### RUN TESTS ###
       - run:
           name: run tests
           command: |
             . venv/bin/activate
-            pip install pytest-cov==2.8.1  # hack to avoid regression
-            python run_tests.py
+            # pip install pytest-cov==2.8.1  # hack to avoid regression
+            #python run_tests.py
+            python -m pytest --cov=xdoctest --cov-config .coveragerc --cov-report term -s
       - store_artifacts:
           path: test-reports
           destination: test-reports
@@ -86,7 +87,7 @@ jobs:
       # ### INITIALIZE AND CACHE REQUIREMENTS ###
       - restore_cache:
           keys:
-          - v2-dependencies-{{ checksum "requirements/runtime.txt" }}-{{ checksum "requirements/optional.txt" }}-{{ checksum "requirements/tests.txt" }}
+          - v3-dependencies-{{ checksum "requirements/runtime.txt" }}-{{ checksum "requirements/optional.txt" }}-{{ checksum "requirements/tests.txt" }}-{{ checksum "requirements/jupyter.txt" }}-{{ checksum "requirements/colors.txt" }}
       - run:
           name: install dependencies
           command: |
@@ -100,14 +101,15 @@ jobs:
       - save_cache:
           paths:
             - ./venv
-          key: v2-dependencies-{{ checksum "requirements/runtime.txt" }}-{{ checksum "requirements/optional.txt" }}-{{ checksum "requirements/tests.txt" }}
+          key: v3-dependencies-{{ checksum "requirements/runtime.txt" }}-{{ checksum "requirements/optional.txt" }}-{{ checksum "requirements/tests.txt" }}-{{ checksum "requirements/jupyter.txt" }}-{{ checksum "requirements/colors.txt" }}
       # ### RUN TESTS ###
       - run:
           name: run tests
           command: |
             . venv/bin/activate
-            pip install pytest-cov==2.8.1  # hack to avoid regression
-            python run_tests.py
+            # pip install pytest-cov==2.8.1  # hack to avoid regression
+            #python run_tests.py
+            python -m pytest --cov=xdoctest --cov-config .coveragerc --cov-report term -s
       - store_artifacts:
           path: test-reports
           destination: test-reports
@@ -227,25 +229,83 @@ jobs:
       - PYTHON_EXE: pypy3
     working_directory: ~/repo-full-pypy3
 
-  heredoc:
-    docker:
-      - image: pypy:3
-    working_directory: ~/dev-only-not-a-real-job
-    steps:
-      - | 
-        __heredoc__=" 
+.__heredoc__: &__heredoc__
+    - | 
+        IMAGE_NAME=circleci/python:3.9
+        docker pull $IMAGE_NAME
+
+        IMAGE_NAME=pypy:3
+        docker pull $IMAGE_NAME
+        docker run -v $HOME/code/xdoctest:/io -it $IMAGE_NAME bash
+
+        IMAGE_NAME=circleci/python:3.10-rc
+        docker pull $IMAGE_NAME
+
+        
         docker run -v $HOME/code/xdoctest:/io -it pypy:3 bash
 
+        docker run -v $HOME/code/xdoctest:/io -it $IMAGE_NAME bash
+
         cd /io
+        
+        # Logic to print out the commands to reproduce CI steps
+        source $HOME/local/init/utils.sh
+        pyblock "
+            import yaml
+            import ubelt as ub
+            data = yaml.safe_load(open(ub.expandpath('$HOME/code/xdoctest/.circleci/config.yml')))
+            JOB_NAME = 'test-minimal-pypy3'
+            job = data['jobs'][JOB_NAME]
+            IMAGE_NAME = job['docker'][0]['image']
+            print('IMAGE_NAME={}'.format(IMAGE_NAME))
+            print('docker run -v $HOME/code/xdoctest:/io -it {} bash'.format(IMAGE_NAME))
+            print(ub.codeblock(
+            '''
+            ###
+            ###
+            # Clone the mounted repo for a fresh start
+            mkdir -p $HOME/code
+            git clone /io /root/{JOB_NAME}
+            cd /root/{JOB_NAME}
+            ''').format(JOB_NAME=JOB_NAME))
+            for kv in job['environment']:
+                for k, v in kv.items():
+                    print('{}={}'.format(k, v))
+            for step in job['steps']:
+                if 'run' in step:
+                    print(step['run']['command'])
+        "
+
 
-        pypy3 -m venv venv 
+        IMAGE_NAME=pypy:3
+        docker run -v /home/joncrall/code/xdoctest:/io -it pypy:3 bash
+        ###
+        ###
+        # Clone the mounted repo for a fresh start
+        mkdir -p /home/joncrall/code
+        git clone /io /root/test-minimal-pypy3
+        cd /root/test-minimal-pypy3
+        PYTHON_EXE=pypy3
+        $PYTHON_EXE -m venv venv || virtualenv -v venv  # first command is python3 || second is python2
         . venv/bin/activate
         # The "minimal" tests install barebones requirements
+        pip install pip -U
         pip install -r requirements/tests.txt
         pip install -r requirements/runtime.txt 
-        pip install -e .
+        pip install .
 
-        pip install pytest-cov==2.8.1  # hack to avoid regression
-        ./run_doctests.sh || echo "pypy failed, but this is allowed"
-        ./run_tests.sh || echo "pypy failed, but this is allowed"
-        "
+        . venv/bin/activate
+        python -m pytest --cov=xdoctest --cov-config .coveragerc --cov-report term -s
+        # pip install pytest-cov==2.8.1  # hack to avoid regression
+        #python run_tests.py
+
+
+        # TO RUN A JOB ON YOUR LOCAL MACHINE
+        # INSTALL CIRCLE CI
+        curl -fLSs https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/master/install.sh | DESTDIR=$HOME/.local/bin bash
+
+        JOB_NAME=test-minimal-pypy3
+        circleci local execute --job $JOB_NAME
+
+        JOB_NAME=test-full-pypy3
+        circleci local execute --job $JOB_NAME

.travis.yml

@@ -24,7 +24,7 @@ python:
   - "3.4"
   - "3.5"
   - "3.6"
-  - "nightly"
+  #- "nightly"  # remove nightly for now
 
 before_install:
   - pip install pip -U

CHANGELOG.md

@@ -5,11 +5,29 @@ We are currently working on porting this changelog to the specifications in
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
-## Version 0.14.1 - Unreleased
+## Version 0.15.0 - Unreleased
+
+
+### Added
+* `pip install xdoctest` can now specify `[colors]` or `[jupyter]`
+* Enhanced REQUIRES directive behavior, multiple comma-separated requirements
+  can now be listed in one directive.
+* Xdoctest can now be run inside of Jupyter notebooks / IPython sessions
+* Xdoctest can now be run on Jupyter notebooks (Note that in general it is
+  better practice to write a module)
 
 ### Fixed
 * Bug in `doctest_callable` where it would not populate globals from the function context.
 
+### Changed
+* Renamed `Config` to `DoctestConfig`
+* Renamed `static_analysis.parse_calldefs` to `static_analysis.parse_static_calldefs`. 
+  A temporary function with the old name is exposed for backwards compatibility.
+* Changed argument name from `modpath_or_name` to `module_identifier` in several functions.
+  This is to better indicate its coercible nature as either a module path, a
+  module name. This change impacts `doctest_module`, `parse_doctestables`,
+  `package_calldefs`. 
+
 
 ## [Version 0.14.0] - Released 2020-08-26 
 

appveyor.yml

@@ -66,7 +66,7 @@ test_script:
   - set PYTHONIOENCODING=utf-8
   - "%PYTHON%\\python.exe -m pip install pytest-cov==2.8.1"
   - "%PYTHON%\\python.exe -m xdoctest xdoctest"
-  - "%PYTHON%\\python.exe -m pytest"
+  - "%PYTHON%\\python.exe -m pytest -s --xdoctest-verbose=3"
 
 after_test:
   # This step builds your wheels.
@@ -77,7 +77,7 @@ after_test:
 
 artifacts:
   # bdist_wheel puts your built wheel in the dist directory
-  - path: dist\*
+  #- path: dist\*
 
 #on_success:
 #  You can use this step to upload your artifacts to a public website.

pytest.ini

@@ -1,9 +1,9 @@
 [pytest]
 # ON COVERAGE OF PYTEST PLUGINS:
 # http://pytest-cov.readthedocs.io/en/latest/plugins.html
-addopts = -p pytester -p no:doctest --xdoctest --ignore-glob=setup.py
+addopts = -p pytester -p no:doctest --xdoctest --ignore-glob=setup.py  --ignore=.tox
 norecursedirs = .git ignore build __pycache__ docs *.egg-info _* dev testing/pybind11_test setup.py
---pyargs --doctest-modules --ignore=.tox
+# --pyargs --doctest-modules --ignore=.tox
 ;rsyncdirs = tox.ini pytest.py _pytest testing
 ;python_files = test_*.py *_test.py testing/*/*.py
 ;python_classes = Test Acceptance

requirements/colors.txt

@@ -0,0 +1,2 @@
+Pygments >= 2.2.0
+colorama >= 0.4.1;platform_system=="Windows"

requirements/jupyter.txt

@@ -0,0 +1,6 @@
+nbformat;python_version>'3.4'
+nbconvert; python_version>'3.4'
+jupyter_client; python_version>'3.4'
+IPython; python_version>'3.4' 
+ipykernel; python_version>'3.4'
+

requirements/optional.txt

@@ -1,2 +1,2 @@
-Pygments >= 2.2.0
-colorama >= 0.4.1;platform_system=="Windows"
+-r colors.txt
+-r jupyter.txt

requirements/tests.txt

@@ -9,9 +9,7 @@ ninja
 pybind11
 
 # for testing doctests in jupyter notebooks
-nbformat
-nbconvert
-jupyter_client
+-r jupyter.txt
 
 #pip uninstall pytest-ipynb
 #pytest-ipynb >= 1.1.1  

setup.py

@@ -25,70 +25,92 @@ def visit_Assign(self, node):
     return visitor.version
 
 
-def parse_requirements(fname='requirements.txt'):
+def parse_requirements(fname='requirements.txt', with_version=False):
     """
     Parse the package dependencies listed in a requirements file but strips
     specific versioning information.
 
-    CommandLine:
-        python -c "import setup; print(setup.parse_requirements())"
+    Args:
+        fname (str): path to requirements file
+        with_version (bool, default=False): if true include version specs
+
+    Returns:
+        List[str]: list of requirements items
     """
-    from os.path import exists
+    from os.path import exists, dirname, join
     import re
     require_fpath = fname
 
-    def parse_line(line):
+    def parse_line(line, dpath=''):
         """
         Parse information from a line in a requirements text file
+
+        line = 'git+https://a.com/somedep@sometag#egg=SomeDep'
+        line = '-e git+https://a.com/somedep@sometag#egg=SomeDep'
         """
+        # Remove inline comments
+        comment_pos = line.find(' #')
+        if comment_pos > -1:
+            line = line[:comment_pos]
+
         if line.startswith('-r '):
             # Allow specifying requirements in other files
-            target = line.split(' ')[1]
+            target = join(dpath, line.split(' ')[1])
             for info in parse_require_file(target):
                 yield info
-        elif line.startswith('-e '):
-            info = {}
-            info['package'] = line.split('#egg=')[1]
-            yield info
         else:
-            # Remove versioning from the package
-            pat = '(' + '|'.join(['>=', '==', '>']) + ')'
-            parts = re.split(pat, line, maxsplit=1)
-            parts = [p.strip() for p in parts]
-
-            info = {}
-            info['package'] = parts[0]
-            if len(parts) > 1:
-                op, rest = parts[1:]
-                if ';' in rest:
+            # See: https://www.python.org/dev/peps/pep-0508/
+            info = {'line': line}
+            if line.startswith('-e '):
+                info['package'] = line.split('#egg=')[1]
+            else:
+                if ';' in line:
+                    pkgpart, platpart = line.split(';')
                     # Handle platform specific dependencies
-                    # http://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-platform-specific-dependencies
-                    version, platform_deps = map(str.strip, rest.split(';'))
-                    info['platform_deps'] = platform_deps
+                    # setuptools.readthedocs.io/en/latest/setuptools.html
+                    # #declaring-platform-specific-dependencies
+                    plat_deps = platpart.strip()
+                    info['platform_deps'] = plat_deps
                 else:
+                    pkgpart = line
+                    platpart = None
+
+                # Remove versioning from the package
+                pat = '(' + '|'.join(['>=', '==', '>']) + ')'
+                parts = re.split(pat, pkgpart, maxsplit=1)
+                parts = [p.strip() for p in parts]
+
+                info['package'] = parts[0]
+                if len(parts) > 1:
+                    op, rest = parts[1:]
                     version = rest  # NOQA
-                info['version'] = (op, version)
+                    info['version'] = (op, version)
             yield info
 
     def parse_require_file(fpath):
+        dpath = dirname(fpath)
         with open(fpath, 'r') as f:
             for line in f.readlines():
                 line = line.strip()
                 if line and not line.startswith('#'):
-                    for info in parse_line(line):
+                    for info in parse_line(line, dpath=dpath):
                         yield info
 
-    # This breaks on pip install, so check that it exists.
-    packages = []
-    if exists(require_fpath):
-        for info in parse_require_file(require_fpath):
-            package = info['package']
-            if not sys.version.startswith('3.4'):
-                # apparently package_deps are broken in 3.4
-                platform_deps = info.get('platform_deps')
-                if platform_deps is not None:
-                    package += ';' + platform_deps
-            packages.append(package)
+    def gen_packages_items():
+        if exists(require_fpath):
+            for info in parse_require_file(require_fpath):
+                parts = [info['package']]
+                if with_version and 'version' in info:
+                    parts.extend(info['version'])
+                if not sys.version.startswith('3.4'):
+                    # apparently package_deps are broken in 3.4
+                    plat_deps = info.get('platform_deps')
+                    if plat_deps is not None:
+                        parts.append(';' + plat_deps)
+                item = ''.join(parts)
+                yield item
+
+    packages = list(gen_packages_items())
     return packages
 
 
@@ -182,6 +204,7 @@ def native_mb_python_tag(plat_impl=None, version_info=None):
 )
 
 
+print(parse_requirements('requirements/tests.txt'))
 if __name__ == '__main__':
     setupkw.update(dict(
         description='A rewrite of the builtin doctest module',
@@ -190,6 +213,8 @@ def native_mb_python_tag(plat_impl=None, version_info=None):
             'all': parse_requirements('requirements.txt'),
             'tests': parse_requirements('requirements/tests.txt'),
             'optional': parse_requirements('requirements/optional.txt'),
+            'colors': parse_requirements('requirements/colors.txt'),
+            'jupyter': parse_requirements('requirements/jupyter.txt'),
         },
         long_description=parse_description(),
         long_description_content_type='text/x-rst',