Upgrade helper for extensions

[blink1073]

Problem

It should be easy for extension authors to upgrade their extensions with support for JupyterLab 3.0.

Proposed Solution

Create a script that extension authors can run that will generate the Python package infrastructure and update package.json with support for dynamic extensions. For extensions that already have a setup.py, we update their package.json and offer a note (and link to example) for updating their setup.py (and pyproject.toml).

[blink1073]

Note: target the extensions-example repo for this script

[blink1073]

@jasongrout while working on this I had a thought. Previously there was an install-ext script in the cookiecutter. Should we re-purpose that to call pip install -e . && jupyter labextension develop --overwrite .?

[blink1073]

Script:

import json
import os
import os.path as osp
from pathlib import Path
import pkg_resources
import shutil
import sys
import subprocess



COOKIECUTTER_BRANCH = "3.0"

# Input is a directory with a package.json or the current directory
# Use the cookiecutter as the source
# Pull in the relevant config
# Pull in the Python parts if possible
# Pull in the scripts if possible
def main(target):
    target = osp.abspath(target)
    package_file = osp.join(target, 'package.json')
    setup_file = osp.join(target, 'setup.py')
    if not osp.exists(package_file):
        raise RuntimeError('No package.json exists in %s' % target)

    # Infer the options from the current directory
    with open(package_file) as fid:
        data = json.load(fid)
    
    if osp.exists(setup_file):
        python_name = subprocess.check_output([sys.executable, 'setup.py', '--name'], cwd=target).decode('utf8').strip()
    else:
        python_name = data['name']
        if '@' in python_name:
            python_name = python_name[1:].replace('/', '_')
    
    arg_data = dict(
        author_name = data.get('author', '<author_name>'),
        labextension_name = data['name'],
        project_short_description = data.get('description', '<description>'),
        has_server_extension = 'y' if osp.exists(setup_file) else 'n',
        has_binder = 'y' if osp.exists(osp.join(target, 'binder')) else 'n',
        repository = data.get('repository', {}).get('url', '<repository'),
        python_name = python_name
    )

    args = ['%s=%s' % (key, value) for (key, value) in arg_data.items()]
    repo = 'https://github.com/jupyterlab/extension-cookiecutter-ts'

    extension_dir = osp.join(target, '_temp_extension')
    if osp.exists(extension_dir):
        shutil.rmtree(extension_dir)

    subprocess.run(['cookiecutter', repo, '--checkout', COOKIECUTTER_BRANCH, '-o', extension_dir] + args, cwd=target)

    python_name = os.listdir(extension_dir)[0]
    extension_dir = osp.join(extension_dir, python_name)

    # From the created package.json, grab the builder dependency
    with open(osp.join(extension_dir, 'package.json')) as fid:
        temp_data = json.load(fid)
    
    for (key, value) in temp_data['devDependencies'].items():
        data['devDependencies'][key] = value

    # Ask the user whether to upgrade the scripts automatically
    warnings = []
    choice = input('overwrite scripts in package.json? [n]: ')
    if choice.upper().startswith('Y'):
        warnings.append('Updated scripts in package.json')
        for (key, value) in temp_data['scripts'].items():
            data['scripts'][key] = value
    else:
        warnings.append('package.json scripts must be updated manually')

    # Set the output directory
    data['jupyterlab']['outputDir'] = python_name + '/static'

    # Look for resolutions in JupyterLab metadata and upgrade those as well
    root_jlab_package = pkg_resources.resource_filename('jupyterlab', 'staging/package.json')
    with open(root_jlab_package) as fid:
        root_jlab_data = json.load(fid)
    
    for (key, value) in root_jlab_data['resolutions'].items():
        if key in data['dependencies']:
            data['dependencies'][key] = value
        if key in data['devDependencies']:
            data['devDependences'][key] = value

    # Sort the entries
    for key in ['scripts', 'dependencies', 'devDependencies']:
        data[key] = dict(sorted(data[key].items()))

    # Update the root package.json file
    with open(package_file, 'w') as fid:
        json.dump(data, fid, indent=2)

    # For the other files, ask about whether to override (when it exists)
    # At the end, list the files that were: added, overridden, skipped
    path = Path(extension_dir)
    for p in path.rglob("*"):
        relpath = osp.relpath(p, path)
        if relpath == "package.json":
            continue
        if p.is_dir():
            continue
        file_target = osp.join(target, relpath)
        if not osp.exists(file_target):
            os.makedirs(osp.dirname(file_target), exist_ok=True)
            shutil.copy(p, file_target)
        else:
            choice = input('overwrite "%s"? [n]: ' % relpath)
            if choice.upper().startswith('Y'):
                shutil.copy(p, file_target)
            else:
                warnings.append('skipped %s' % relpath)

    # Print out all warnings
    for warning in warnings:
        print('**', warning)

    print('** Remove _temp_extensions directory when finished')


if __name__ == "__main__":
    if len(sys.argv) > 1:
        main(sys.argv[1])
    else:
        main(os.getcwd())

[blink1073]

The script is done, the next question is where it should live. It is generic enough that it could live as python -m jupyterlab.upgrade_extension.

[jasongrout]

Nice! if it's generic enough to be broadly useful, I'm happy to have it live as a standalone module in jlab.

[blink1073]

Cool, I think it just needs detection of whether the file to be overwritten is actually changing content, I'll do that as part of the PR.