Fix linking to editable install on Windows

On Windows during compile time it is not only target .dll that is needed
to link to a DSO, but also corresponding .lib and .exp files. And if the
.lib file cannot be found linking fails. On regular install for a DSO X we are
currently installing all X.dll, X.lib and X.exp, but for in-place builds we
copy only X.dll into intree without X.lib and the rest.

This leads to build failures when an external project tries to link to a DSO
from `pip install -e` installed project.  Below is, for example, how it looks
for the link failure of added example/project2 to dsodemo.lib.demo without the fix:

```console
(1.wenv) Z:\home\kirr\src\tools\go\pygo-win\setuptools_dso\example\project2>python setup.py build_ext -i
running build_ext
building 'use_dsodemo.ext' extension
Z:\home\kirr\src\tools\go\pygo-win\BuildTools\vc\tools\msvc\14.35.32215\bin\Hostx64\x64\cl.exe /c /nologo /Od /W3 /GL /DNDEBUG /MD -I../src -IZ:\ho
me\kirr\src\tools\go\pygo-win\1.wenv\include "-IC:\Program Files\Python310\include" "-IC:\Program Files\Python310\Include" -Iz:\home\kirr\src\tools
\go\pygo-win\BuildTools\vc\tools\msvc\14.35.32215\include -Iz:\home\kirr\src\tools\go\pygo-win\BuildTools\kits\10\include\10.0.22000.0\shared -Iz:\
home\kirr\src\tools\go\pygo-win\BuildTools\kits\10\include\10.0.22000.0\ucrt -Iz:\home\kirr\src\tools\go\pygo-win\BuildTools\kits\10\include\10.0.2
2000.0\um -Iz:\home\kirr\src\tools\go\pygo-win\BuildTools\kits\10\include\10.0.22000.0\winrt /EHsc /Tpsrc/use_dsodemo/ext.cpp /Fobuild\temp.win-amd
64-cpython-310\Release\src/use_dsodemo/ext.obj
ext.cpp

creating Z:\home\kirr\src\tools\go\pygo-win\setuptools_dso\example\project2\build\lib.win-amd64-cpython-310
creating Z:\home\kirr\src\tools\go\pygo-win\setuptools_dso\example\project2\build\lib.win-amd64-cpython-310\use_dsodemo
Z:\home\kirr\src\tools\go\pygo-win\BuildTools\vc\tools\msvc\14.35.32215\bin\Hostx64\x64\link.exe /nologo /INCREMENTAL:NO /LTCG /DLL /MANIFEST:EMBED
,ID=2 /MANIFESTUAC:NO /LIBPATH:Z:\home\kirr\src\tools\go\pygo-win\setuptools_dso\example\src\dsodemo\lib "/LIBPATH:C:\Program Files\Python310\libs"
 "/LIBPATH:C:\Program Files\Python310" /LIBPATH:z:\home\kirr\src\tools\go\pygo-win\BuildTools\vc\tools\msvc\14.35.32215\lib\x64 /LIBPATH:z:\home\ki
rr\src\tools\go\pygo-win\BuildTools\kits\10\lib\10.0.22000.0\ucrt\x64 /LIBPATH:z:\home\kirr\src\tools\go\pygo-win\BuildTools\kits\10\lib\10.0.22000
.0\um\x64 demo.lib /EXPORT:PyInit_ext build\temp.win-amd64-cpython-310\Release\src/use_dsodemo/ext.obj /OUT:build\lib.win-amd64-cpython-310\use_dso
demo\ext.cp310-win_amd64.pyd /IMPLIB:build\temp.win-amd64-cpython-310\Release\src/use_dsodemo\ext.cp310-win_amd64.lib

LINK : fatal error LNK1181: cannot open input file 'demo.lib'
error: command 'Z:\\home\\kirr\\src\\tools\\go\\pygo-win\\BuildTools\\vc\\tools\\msvc\\14.35.32215\\bin\\Hostx64\\x64\\link.exe' failed with exit code 1181
```

This fix is simple: on inplace build copy into intree not only .dll but
also .lib and .exp files.

.github/workflows/build.yml

@@ -84,21 +84,32 @@ jobs:
           python setup.py clean -a
           git clean -fdx .
           python -m pip install -v --no-index -f ../dist .
-          cd ..
+          cd project2
+          # install project2.
+          # --no-build-isolation is needed so that ^^^ installed dsodemo could be found.
+          # --no-use-pep517 is used to workaround https://github.com/pypa/setuptools/issues/1694 on py36 and py35
+          python -m pip install -v --no-index -f ../dist wheel  # needed by vvv
+          python -m pip install -v --no-index --no-build-isolation --no-use-pep517 -f ../../dist .
+          cd ../..
           python -m dsodemo.cli
           python -m nose2 dsodemo
+          python -m use_dsodemo.cli
       - name: Test Example inplace
         shell: bash
         run: |
           set -x
           cd example
           python setup.py clean -a
           git clean -fdx .
+          export PYTHONPATH="`pwd`/src"
           python setup.py -v build_dso -i
           python setup.py -v build_dso -i -f  # incremental recompile
           python setup.py -v build_ext -i
-          (cd src && python -m dsodemo.cli)
-          cd ..
+          cd project2
+          python setup.py -v build_ext -i
+          cd ../..
+          python -m dsodemo.cli
+          (cd example/project2/src && python -m use_dsodemo.cli)
       - name: Test Example DSO only
         shell: bash
         run: |

example/project2/MANIFEST.in

@@ -0,0 +1,4 @@
+include pyproject.toml
+include src/*.h
+include src/*.c
+include src/*.cpp

example/project2/pyproject.toml

@@ -0,0 +1,2 @@
+[build-system]
+requires = ["setuptools", "wheel", "setuptools_dso"]

example/project2/setup.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+"""Project2 demonstrates how to link-to dsodemo and use it from external project"""
+
+from setuptools_dso import Extension, setup
+from os.path import dirname, join, abspath
+
+import dsodemo.lib
+
+ext = Extension('use_dsodemo.ext', ['src/use_dsodemo/ext.cpp'],
+    dsos=['dsodemo.lib.demo'],
+    include_dirs=[dirname(dsodemo.lib.__file__)],  # TODO automatically discover it like we do for library_dirs
+)
+setup(
+    name='use_dsodemo',
+    version="0.1",
+    install_requires = ['setuptools_dso', 'dsodemo'],
+    packages=['use_dsodemo'],
+    package_dir={'': 'src'},
+    ext_modules = [ext],
+)

example/project2/src/use_dsodemo/cli.py

@@ -0,0 +1,13 @@
+from __future__ import print_function
+
+import dsodemo.ext.dtest  # preload dsodemo.lib.demo dso which dsodemo.ext.dtest uses
+
+from . import ext
+
+def main():
+    print('via .ext -> dsodemo.lib.demo:')
+    print(ext.dsodemo_foo())
+    print(ext.dsodemo_bar())
+
+if __name__ == '__main__':
+    main()

example/project2/src/use_dsodemo/ext.cpp

@@ -0,0 +1,58 @@
+#include <Python.h>
+
+#include "mylib.h"
+
+static
+PyObject* call_dsodemo_foo(PyObject *junk)
+{
+    return PyUnicode_FromString(foo());
+}
+
+static
+PyObject* call_dsodemo_bar(PyObject *junk)
+{
+    return PyUnicode_FromString(bar().c_str());
+}
+
+static struct PyMethodDef use_dsodemo_methods[] = {
+    {"dsodemo_foo", (PyCFunction)call_dsodemo_foo, METH_NOARGS,
+     "dsodemo_foo() -> unicode\n"
+     "call foo"},
+    {"dsodemo_bar", (PyCFunction)call_dsodemo_bar, METH_NOARGS,
+     "dsodemo_bar() -> unicode\n"
+     "call bar"},
+    {NULL}
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef use_dsodemo_module = {
+  PyModuleDef_HEAD_INIT,
+    "use_dsodemo.ext",
+    NULL,
+    -1,
+    use_dsodemo_methods,
+};
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+#  define PyMOD(NAME) PyObject* PyInit_##NAME (void)
+#else
+#  define PyMOD(NAME) void init##NAME (void)
+#endif
+
+extern "C"
+PyMOD(ext)
+{
+#if PY_MAJOR_VERSION >= 3
+        PyObject *mod = PyModule_Create(&use_dsodemo_module);
+#else
+        PyObject *mod = Py_InitModule("use_dsodemo.ext", use_dsodemo_methods);
+#endif
+        if(mod) {
+        }
+#if PY_MAJOR_VERSION >= 3
+    return mod;
+#else
+    (void)mod;
+#endif
+}

src/setuptools_dso/dsocmd.py

@@ -394,7 +394,9 @@ def build_dso(self, dso):
             # The .lib is considered "temporary" for extensions, but not for us
             # so we pass export_symbols=None and put it along side the .dll
             # eg. "pkg\mod\mylib.dll" and "pkg\mod\mylib.lib"
-            extra_args.append('/IMPLIB:%s.lib'%(os.path.splitext(outlib)[0]))
+            outlib_lib = '%s.lib' % os.path.splitext(outlib)[0]
+            outlib_exp = '%s.exp' % os.path.splitext(outlib)[0]
+            extra_args.append('/IMPLIB:%s'%outlib_lib)
 
         elif baselib!=solib: # ELF
             extra_args.extend(['-Wl,-h,%s'%solibbase])
@@ -428,13 +430,17 @@ def build_dso(self, dso):
             pkg = '.'.join(dso.name.split('.')[:-1])    # path.to.dso -> path.to
             pkgdir = build_py.get_package_dir(pkg)      # path.to -> src/path/to
 
-            solib_dst   = os.path.join(pkgdir, os.path.basename(solib))     # path/to/dso.so -> src/path/to/dso.so
-            baselib_dst = os.path.join(pkgdir, os.path.basename(baselib))
+            def inplace_dst(path): # build/.../path/to/dso.so -> src/path/to/dso.so
+                return os.path.join(pkgdir, os.path.basename(path))
 
-            self.mkpath(os.path.dirname(solib_dst))
-            self.copy_file(outlib, solib_dst)
+            self.mkpath(os.path.dirname(inplace_dst(outlib)))
+            self.copy_file(outlib, inplace_dst(outlib))
             if baselib!=solib:
-                self.copy_file(outbaselib, baselib_dst)
+                self.copy_file(outbaselib, inplace_dst(outbaselib))
+            if sys.platform == "win32":
+                # on windows linking to x.dll goes through x.lib and x.exp first
+                self.copy_file(outlib_lib, inplace_dst(outlib_lib))
+                self.copy_file(outlib_exp, inplace_dst(outlib_exp))
 
     def gen_info_module(self, dso):
         if not dso.gen_info:

testme.sh

@@ -28,33 +28,49 @@ cd example
 python setup.py clean -a
 git clean -fdx	# `setup.py clean` does not clean inplace built files
 (cd src && python -m dsodemo.cli 2>/dev/null) && die "error: worktree must be clean"
+(cd project2/src && python -m use_dsodemo.cli 2>/dev/null) && die "error: worktree must be clean"
 python setup.py -v build_dso -i
 python setup.py -v build_dso -i -f  # incremental recompile
 python setup.py -v build_ext -i
+X="`pwd`/src"
+cd project2
+PYTHONPATH=$X python setup.py -v build_ext -i
+cd ..
 (cd src && python -m dsodemo.cli)
+(cd project2/src && PYTHONPATH=$X python -m use_dsodemo.cli)
 
 
 # build + install
 echo -e '\n* build + install\n'
 python setup.py clean -a
 git clean -fdx
 python -m dsodemo.cli 2>/dev/null && die "error: worktree must be clean"
+python -m use_dsodemo.cli 2>/dev/null && die "error: worktree must be clean"
 pip install --no-build-isolation -v .
+# --no-use-pep517 is used to workaround https://github.com/pypa/setuptools/issues/1694 on py36 and py35
+cd project2
+pip install --no-build-isolation --no-use-pep517 -v .
 
-cd ..
-
+cd ../..
 python -m dsodemo.cli
+python -m use_dsodemo.cli
 
 
 # install in development mode
 pip uninstall -y dsodemo
+pip uninstall -y use_dsodemo
 python -m dsodemo.cli 2>/dev/null && die "error: dsodemo not uninstalled"
+python -m use_dsodemo.cli 2>/dev/null && die "error: dsodemo not uninstalled"
 
 cd example
 python setup.py clean -a
 git clean -fdx
 (cd src && python -m dsodemo.cli 2>/dev/null) && die "error: worktree must be clean"
+(cd project2/src && python -m use_dsodemo.cli 2>/dev/null) && die "error: worktree must be clean"
 pip install --no-build-isolation -v -e .
+cd project2
+pip install --no-build-isolation --no-use-pep517 -v -e .
 
-cd ..
+cd ../..
 python -m dsodemo.cli
+python -m use_dsodemo.cli