chore: fix editable build

Fixes #270.

.github/actions/setup-firefox/action.yml

@@ -0,0 +1,19 @@
+name: "Setup firefox"
+description: "Sets up firefox on Linux"
+runs:
+  using: "composite"
+  steps:
+  - run: |
+      sudo add-apt-repository ppa:mozillateam/ppa
+      echo '
+      Package: *
+      Pin: release o=LP-PPA-mozillateam
+      Pin-Priority: 1001
+
+      Package: firefox
+      Pin: version 1:1snap1-0ubuntu2
+      Pin-Priority: -1
+      ' | sudo tee /etc/apt/preferences.d/mozilla-firefox
+      sudo apt-get install firefox xvfb
+    if: startsWith(runner.os, 'Linux')
+    shell: bash

.github/workflows/build.yml

@@ -32,19 +32,7 @@ jobs:
       run: npm run build-module --no-typecheck
     - name: Build Python client bundles
       run: npm run build-python --no-typecheck
-    - run: |
-        sudo add-apt-repository ppa:mozillateam/ppa
-        echo '
-        Package: *
-        Pin: release o=LP-PPA-mozillateam
-        Pin-Priority: 1001
-
-        Package: firefox
-        Pin: version 1:1snap1-0ubuntu2
-        Pin-Priority: -1
-        ' | sudo tee /etc/apt/preferences.d/mozilla-firefox
-        sudo apt-get install firefox xvfb
-      if: startsWith(runner.os, 'Linux')
+    - uses: ./.github/actions/setup-firefox
     - name: Run JavaScript tests (including WebGL)
       # Swiftshader, used by Chrome headless, crashes when running Neuroglancer
       # tests.
@@ -68,9 +56,10 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - '3.7'
           - '3.8'
           - '3.9'
+          - '3.10'
+          - '3.11'
         node-version:
           - '16.x'
         os:
@@ -106,32 +95,18 @@ jobs:
     # Uncomment the action below for an interactive shell
     # - name: Setup tmate session
     #   uses: mxschmitt/action-tmate@v3
-    - run: npm install
-    - run: npm run build-python -- --no-typecheck
     - name: Install Python packaging/test tools
       run: python -m pip install --upgrade pip tox wheel numpy pytest
-    - run: |
-        sudo add-apt-repository ppa:mozillateam/ppa
-        echo '
-        Package: *
-        Pin: release o=LP-PPA-mozillateam
-        Pin-Priority: 1001
-
-        Package: firefox
-        Pin: version 1:1snap1-0ubuntu2
-        Pin-Priority: -1
-        ' | sudo tee /etc/apt/preferences.d/mozilla-firefox
-        sudo apt-get install firefox xvfb
-      if: startsWith(runner.os, 'Linux')
+    - uses: ./.github/actions/setup-firefox
     - name: Test with tox
       run: tox -e ${{ fromJSON('["skip-browser-tests","firefox-xvfb"]')[runner.os == 'Linux'] }}
       env:
         TOX_TESTENV_PASSENV: GH_TOKEN
         GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     # Verify that editable install works
     - name: Install in editable form
-      run: pip install -e .
-    - name: Run tests against editable install
+      run: pip install -e . --config-settings editable_mode=strict
+    - name: Run Python tests against editable install (excluding WebGL)
       working-directory: python/tests
       run: pytest -vv --skip-browser-tests
 

config/esbuild-cli.js

@@ -81,9 +81,11 @@ async function main(argv) {
   let minify = true;
   let python = false;
   let moduleBuild = false;
-  let outDir = undefined;
+  let outDir = argv.output;
   let id = argv.config;
-  const pythonOutDir = path.resolve(__dirname, '..', 'python', 'neuroglancer', 'static');
+  const pythonOutDir = argv.output !== undefined ?
+      argv.output :
+      path.resolve(__dirname, '..', 'python', 'neuroglancer', 'static');
 
   switch (id) {
     case 'min':
@@ -119,7 +121,7 @@ async function main(argv) {
     googleTagManager: argv.googleTagManager,
     analyze: argv.analyze,
   });
-  if (moduleBuild) {
+  if (moduleBuild && argv.output === undefined) {
     try {
       if ((await fs.promises.lstat(builder.outDir)).isDirectory()) {
         await fs.promises.rmdir(builder.outDir, {recursive: true});
@@ -205,6 +207,11 @@ if (require.main === module) {
               description: 'Additional JSON/JavaScript config file to load.',
               configParser: x => mungeConfig(require(x)),
             },
+            output: {
+              type: 'string',
+              nargs: 1,
+              description: 'Output directory.',
+            },
             ['google-tag-manager']: {
               group: 'Customization',
               type: 'string',

pyproject.toml

@@ -1,2 +1,2 @@
 [build-system]
-requires = ["setuptools>=30.3.0", "wheel", "setuptools_scm", "oldest-supported-numpy"]
+requires = ["setuptools>=64", "wheel", "setuptools_scm", "oldest-supported-numpy"]

python/build_tools/cibuildwheel.sh

@@ -5,6 +5,9 @@ export CIBW_SKIP="cp27-* cp36-* pp* *_i686 *-win32"
 export CIBW_TEST_EXTRAS="test"
 export CIBW_TEST_COMMAND="python -m pytest {project}/python/tests -vv -s --skip-browser-tests"
 export CIBW_MANYLINUX_X86_64_IMAGE=manylinux2014
+export CIBW_ENVIRONMENT_PASS_LINUX="NEUROGLANCER_BUILD_BUNDLE_INPLACE"
+
+export NEUROGLANCER_BUILD_BUNDLE_INPLACE=1
 
 # Skip testing on musllinux due to pillow binary wheels not being available.
 export CIBW_TEST_SKIP="*-musllinux*"

setup.py

@@ -12,9 +12,11 @@
 import distutils.command.build
 import os
 import platform
+import shutil
 import subprocess
 import tempfile
 import time
+import setuptools.command.build
 import setuptools.command.build_ext
 import setuptools.command.develop
 import setuptools.command.install
@@ -27,18 +29,32 @@
 src_dir = os.path.join(python_dir, 'ext', 'src')
 openmesh_dir = os.path.join(python_dir, 'ext', 'third_party', 'openmesh', 'OpenMesh', 'src')
 
+CLIENT_FILES = [
+    "index.html",
+    "main.bundle.js",
+    "main.bundle.css",
+    "main.bundle.js.map",
+    "main.bundle.css.map",
+    "chunk_worker.bundle.js",
+    "chunk_worker.bundle.js.map",
+    "async_computation.bundle.js",
+    "async_computation.bundle.js.map",
+]
+
 with open(os.path.join(python_dir, 'README.md'), mode='r', encoding='utf-8') as f:
     long_description = f.read()
 
 
-def _maybe_bundle_client(cmd):
+def _maybe_bundle_client(cmd, inplace=False):
     """Build the client bundle if it does not already exist.
 
     If it has already been built but is stale, the user is responsible for
     rebuilding it.
     """
 
     bundle_client_cmd = cmd.distribution.get_command_obj('bundle_client')
+    if inplace:
+        bundle_client_cmd.build_bundle_inplace = True
     if bundle_client_cmd.skip_rebuild is None:
         bundle_client_cmd.skip_rebuild = True
     cmd.run_command('bundle_client')
@@ -60,11 +76,12 @@ def _setup_temp_egg_info(cmd):
 
 
 class SdistCommand(setuptools.command.sdist.sdist):
+
     def run(self):
         # Build the client bundle if it does not already exist.  If it has
         # already been built but is stale, the user is responsible for
         # rebuilding it.
-        _maybe_bundle_client(self)
+        _maybe_bundle_client(self, inplace=True)
         _setup_temp_egg_info(self)
         super().run()
 
@@ -76,22 +93,11 @@ def make_release_tree(self, base_dir, files):
         super().make_release_tree(base_dir, files)
 
 
-class BuildCommand(distutils.command.build.build):
-    def finalize_options(self):
-        if self.build_base == 'build':
-            # Use temporary directory instead, to avoid littering the source directory
-            # with a `build` sub-directory.
-            tempdir = tempfile.TemporaryDirectory()
-            self.build_base = tempdir.name
-            atexit.register(tempdir.cleanup)
-        super().finalize_options()
-
-    def run(self):
-        _maybe_bundle_client(self)
-        super().run()
+setuptools.command.build.build.sub_commands.append(('bundle_client', None))
 
 
 class BuildExtCommand(setuptools.command.build_ext.build_ext):
+
     def finalize_options(self):
         super().finalize_options()
         # Prevent numpy from thinking it is still in its setup process
@@ -104,23 +110,28 @@ def finalize_options(self):
 
 
 class InstallCommand(setuptools.command.install.install):
+
     def run(self):
         _setup_temp_egg_info(self)
         super().run()
 
 
 class DevelopCommand(setuptools.command.develop.develop):
+
     def run(self):
         _maybe_bundle_client(self)
         super().run()
 
 
-class BundleClientCommand(distutils.command.build.build):
+class BundleClientCommand(setuptools.command.build.build, setuptools.command.build.SubCommand):
+
+    editable_mode: bool = False
 
     user_options = [
         ('client-bundle-type=', None,
          'The nodejs bundle type. "min" (default) creates condensed static files for production, "dev" creates human-readable files.'
          ),
+        ('build-bundle-inplace', None, 'Build the client bundle inplace.'),
         ('skip-npm-reinstall', None,
          'Skip running `npm install` if the `node_modules` directory already exists.'),
         ('skip-rebuild', None,
@@ -129,25 +140,64 @@ class BundleClientCommand(distutils.command.build.build):
 
     def initialize_options(self):
 
+        self.build_lib = None
         self.client_bundle_type = 'min'
         self.skip_npm_reinstall = None
         self.skip_rebuild = None
+        self.build_bundle_inplace = None
 
     def finalize_options(self):
+        self.set_undefined_options("build_py", ("build_lib", "build_lib"))
 
         if self.client_bundle_type not in ['min', 'dev']:
             raise RuntimeError('client-bundle-type has to be one of "min" or "dev"')
 
         if self.skip_npm_reinstall is None:
             self.skip_npm_reinstall = False
 
+        if self.build_bundle_inplace is None:
+            self.build_bundle_inplace = (os.getenv('NEUROGLANCER_BUILD_BUNDLE_INPLACE') == '1')
+
         if self.skip_rebuild is None:
-            self.skip_rebuild = False
+            self.skip_rebuild = self.build_bundle_inplace
+
+    def get_outputs(self):
+        if self.editable_mode or self.build_bundle_inplace:
+            return []
+        build_lib = self.build_lib
+        return [f"{build_lib}/{f}" for f in self._get_bundle_files()]
+
+    def _get_bundle_files(self):
+        return [f"neuroglancer/static/{f}" for f in CLIENT_FILES]
+
+    def get_source_files(self):
+        return []
+
+    def get_output_mapping(self):
+        return {}
 
     def run(self):
+        inplace = self.editable_mode or self.build_bundle_inplace
+        print(f'Building client bundle: inplace={inplace}, skip_rebuild={self.skip_rebuild}')
+
+        # If building from an sdist, `package.json` won't be present but the
+        # bundled files will.
+        if not os.path.exists(os.path.join(root_dir, 'package.json')):
+            print('Skipping build of client bundle because package.json does not exist')
+            for dest, source in self.get_output_mapping().items():
+                if dest != source:
+                    shutil.copyfile(source, dest)
+            return
+
+        if inplace:
+            output_base_dir = python_dir
+        else:
+            output_base_dir = self.build_lib
 
-        if self.skip_rebuild:
-            html_path = os.path.join(python_dir, 'neuroglancer', 'static', 'index.html')
+        output_dir = os.path.join(output_base_dir, 'neuroglancer', 'static')
+
+        if self.skip_rebuild and inplace:
+            html_path = os.path.join(output_dir, 'index.html')
             if os.path.exists(html_path):
                 print('Skipping rebuild of client bundle since %s already exists' % (html_path, ))
                 return
@@ -161,7 +211,7 @@ def run(self):
                 print('Skipping `npm install` since %s already exists' % (node_modules_path, ))
             else:
                 subprocess.call('npm i', shell=True, cwd=root_dir)
-            res = subprocess.call('npm run %s' % t, shell=True, cwd=root_dir)
+            res = subprocess.call(f'npm run {t} -- --output={output_dir}', shell=True, cwd=root_dir)
         except:
             raise RuntimeError(
                 'Could not run \'npm run %s\'. Make sure node.js >= v12 is installed and in your path.'
@@ -197,6 +247,7 @@ def run(self):
 if platform.system() == 'Windows':
     extra_compile_args.append('/d2FH4-')
 
+
 # Copied from setuptools_scm, can be removed once a released version of
 # setuptools_scm supports `version_scheme=no-guess-dev`.
 #
@@ -273,7 +324,6 @@ def _no_guess_dev_version(version):
     ],
     cmdclass={
         'sdist': SdistCommand,
-        'build': BuildCommand,
         'bundle_client': BundleClientCommand,
         'build_ext': BuildExtCommand,
         'install': InstallCommand,

tox.ini

@@ -15,9 +15,6 @@ extras =
   test
   chrome,firefox: test-browser
 
-whitelist_externals =
-  xvfb-run
-
 # Pass through DISPLAY to allow non-headless web browsers on Linux
 passenv = DISPLAY