Rebase from 'debian/melodic/catkin_virtualenv'

locusrobotics · Aug 24, 2020 · 86f74df · 86f74df
1 parent feaf047
commit 86f74df
Show file tree

Hide file tree

Showing 39 changed files with 870 additions and 965 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -2,19 +2,74 @@
 Changelog for package catkin_virtualenv
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-0.5.2 (2020-02-05)
-------------------
-* First python2 issue of 2020 (#49)
+0.6.1 (2020-08-24)
+------------------
+* Correct dependencies and autoformat (`#72 <https://github.com/locusrobotics/catkin_virtualenv/issues/72>`_)
+  * Remove python-virtualenv dep
+  * Add python2-dev
+  * Lint
+* We're ok with any 44.x version of setuptools (`#71 <https://github.com/locusrobotics/catkin_virtualenv/issues/71>`_)
+  But not anything newer.
+  Older versions don't appear to work reliably with `pip==20.1`.
+  This helps when running a build of a package depending on catkin_virtualenv on OS which ship with an old version of setuptools (such as Ubuntu Xenial) when `USE_SYSTEM_PACKAGES` is not set to `FALSE`. In that situation, only specifying 'setuptools<45` will be true, as setuptools is installed (in the systems site packages), so pip will not upgrade it. Specifying a minimum version like this will force pip to always install an up-to-date version.
+* Contributors: G.A. vd. Hoorn, Paul Bovbel
+
+0.6.0 (2020-07-14)
+------------------
+* Don't require catkin_package to be called before catkin_generate_virtualenv (`#67 <https://github.com/locusrobotics/catkin_virtualenv/issues/67>`_)
+* Revert "Downgrade docutils so that boto works (`#66 <https://github.com/locusrobotics/catkin_virtualenv/issues/66>`_)"
+  This reverts commit 998cd6add2e43e12036d0db15a7c4d58fe3411cf.
+* Downgrade docutils so that boto works (`#66 <https://github.com/locusrobotics/catkin_virtualenv/issues/66>`_)
+  See https://github.com/boto/botocore/issues/1942 and related threads.
+* Make regex for Python bytecode more selective (`#65 <https://github.com/locusrobotics/catkin_virtualenv/issues/65>`_)
+  Fix regex to match only files ending in ".py[co]" and not files ending
+  in "py[co]".
+* Remove user specific paths (`#63 <https://github.com/locusrobotics/catkin_virtualenv/issues/63>`_)
+  * Remove user specific paths
+  * Change working directory of venv_lock command
+  * Lock catkin_virtualenv base requirements
+  Co-authored-by: Paul Bovbel <paul@bovbel.com>
+* RST-3172 Check that requirements file is locked (`#62 <https://github.com/locusrobotics/catkin_virtualenv/issues/62>`_)
+* Two helpful hints (`#61 <https://github.com/locusrobotics/catkin_virtualenv/issues/61>`_)
+* Fix input requirements warning (`#58 <https://github.com/locusrobotics/catkin_virtualenv/issues/58>`_)
+  * Only warn about INPUT_REQUIREMENTS if a package exports requirements to begin with
+  * Update catkin_virtualenv/cmake/catkin_generate_virtualenv.cmake
+  Co-authored-by: Andrew Blakey <ablakey@gmail.com>
+  Co-authored-by: Andrew Blakey <ablakey@gmail.com>
+* Preserve symlinks during copy (`#57 <https://github.com/locusrobotics/catkin_virtualenv/issues/57>`_)
+* Don't ignore unknown args
+* RST-3172 Refactor catkin_virtualenv to allow locking dependencies (`#55 <https://github.com/locusrobotics/catkin_virtualenv/issues/55>`_)
+  * Remove unused options
+  * Fix regex for comments
+  * Migrate scripts
+  * Remove old code
+  * Move common requirements to an export file
+  * Minor cleanup
+  * Remove requirement-parsing unit tests
+  * Fix logging config
+  * Fix test builds
+  * Generate lock files
+  * Fix tests
+  * Move dh-virtualenv functions into separate file
+  * Fix roslint
+  * Update docs
+  * Update requirements
+  * CMake comments
+  * Fix pip-args
+  * README fixup
+  * Correct ARG_LOCK_FILE handling
+  * Remove headers
+  * Use set comprehension
+  * Add migration doc
+  * Respin
+* Use exec to dive into python (`#51 <https://github.com/locusrobotics/catkin_virtualenv/issues/51>`_)
+* First python2 issue of 2020 (`#49 <https://github.com/locusrobotics/catkin_virtualenv/issues/49>`_)
   * Clean up options, virtualenv installs setuptools by default
   * Make sure we install a compatible setuptools version for py2 venv
-* Contributors: Paul Bovbel
-
-0.5.1 (2019-08-19)
-------------------
 * catkin-pkg-modules has disappeared off pypi (`#46 <https://github.com/locusrobotics/catkin_virtualenv/issues/46>`_)
   * catkin-pkg-modules has disappeared off pypi, but catkin-pkg is still there
   * Version all requirements
-* Contributors: Paul Bovbel
+* Contributors: David V. Lu!!, Michael Johnson, Paul Bovbel, abencz
 
 0.5.0 (2019-06-21)
 ------------------

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -12,18 +12,24 @@ install(DIRECTORY cmake
   PATTERN ${PROJECT_NAME}-extras.cmake* EXCLUDE
 )
 
-catkin_install_python(
-  PROGRAMS
-    scripts/build_venv
-    scripts/combine_requirements
-    scripts/glob_requirements
-  DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
+set(python_scripts
+  scripts/collect_requirements
+  scripts/venv_init
+  scripts/venv_check
+  scripts/venv_lock
+  scripts/venv_install
+  scripts/venv_relocate
 )
 
+catkin_install_python(PROGRAMS ${python_scripts}
+  DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION})
+
+install(FILES requirements.txt
+  DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION})
+
 if(CATKIN_ENABLE_TESTING)
   find_package(roslint REQUIRED)
   roslint_python()
+  roslint_python(${python_scripts})
   roslint_add_test()
-
-  catkin_add_nosetests(test)
 endif()
diff --git a/cmake/catkin_generate_virtualenv.cmake b/cmake/catkin_generate_virtualenv.cmake
@@ -17,42 +17,50 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 function(catkin_generate_virtualenv)
-  set(oneValueArgs PYTHON_VERSION PYTHON_VERSION_MAJOR USE_SYSTEM_PACKAGES ISOLATE_REQUIREMENTS)
+  set(oneValueArgs PYTHON_VERSION PYTHON_INTERPRETER USE_SYSTEM_PACKAGES ISOLATE_REQUIREMENTS INPUT_REQUIREMENTS CHECK_VENV)
   set(multiValueArgs EXTRA_PIP_ARGS)
   cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} )
 
-  # Check if this package already has a virtualenv target before creating one
-  if(TARGET ${PROJECT_NAME}_generate_virtualenv)
-    message(WARNING "catkin_generate_virtualenv was called twice")
-    return()
-  endif()
+  ### Handle argument defaults and deprecations
 
-  # Backwards compatibility for PYTHON_VERSION_MAJOR, overriding PYTHON_VERSION
-  if(DEFINED ARG_PYTHON_VERSION_MAJOR)
-    set(ARG_PYTHON_VERSION ${ARG_PYTHON_VERSION_MAJOR})
-    message(WARNING "PYTHON_VERSION_MAJOR has been deprecated, please set PYTHON_VERSION instead")
+  if(DEFINED ARG_PYTHON_VERSION)
+    message(WARNING "PYTHON_VERSION has been deprecated, set 'PYTHON_INTERPRETER python${ARG_PYTHON_VERSION}' instead")
+    set(ARG_PYTHON_INTERPRETER "python${ARG_PYTHON_VERSION}")
   endif()
 
-  if(NOT DEFINED ARG_PYTHON_VERSION)
-    set(ARG_PYTHON_VERSION 2)
+  if(NOT DEFINED ARG_PYTHON_INTERPRETER)
+    set(ARG_PYTHON_INTERPRETER "python2")
   endif()
 
-  if(NOT DEFINED ARG_USE_SYSTEM_PACKAGES)
-    set(ARG_USE_SYSTEM_PACKAGES TRUE)
+  if(NOT DEFINED ARG_USE_SYSTEM_PACKAGES OR ARG_USE_SYSTEM_PACKAGES)
+    message(STATUS "Using system site packages")
+    set(venv_args "--use-system-packages")
   endif()
 
-  if(NOT DEFINED ARG_ISOLATE_REQUIREMENTS)
-    set(ARG_ISOLATE_REQUIREMENTS FALSE)
+  if(ARG_ISOLATE_REQUIREMENTS)
+    message(STATUS "Only using requirements from this catkin package")
+    set(lock_args "${lock_args} --no-deps")
   endif()
 
   if (NOT DEFINED ARG_EXTRA_PIP_ARGS)
-    set(ARG_EXTRA_PIP_ARGS "-qq")
+    set(ARG_EXTRA_PIP_ARGS "-qq" "--retries 10" "--timeout 30")
   endif()
+
   # Convert CMake list to ' '-separated list
   string(REPLACE ";" "\ " processed_pip_args "${ARG_EXTRA_PIP_ARGS}")
   # Double-escape needed to get quote down through cmake->make->shell layering
   set(processed_pip_args \\\"${processed_pip_args}\\\")
 
+  # Check if this package already has a virtualenv target before creating one
+  if(TARGET ${PROJECT_NAME}_generate_virtualenv)
+    message(WARNING "catkin_generate_virtualenv was called twice")
+    return()
+  endif()
+
+  # Make sure CATKIN_* paths are initialized
+  catkin_destinations()  # oh the places we'll go
+
+  ### Start building virtualenv
   set(venv_dir venv)
 
   set(venv_devel_dir ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/${venv_dir})
@@ -61,73 +69,104 @@ function(catkin_generate_virtualenv)
   set(${PROJECT_NAME}_VENV_DEVEL_DIR ${venv_devel_dir} PARENT_SCOPE)
   set(${PROJECT_NAME}_VENV_INSTALL_DIR ${venv_install_dir} PARENT_SCOPE)
 
-  if(${ARG_ISOLATE_REQUIREMENTS})
-    message(STATUS "Only using requirements from this catkin package")
-    set(glob_args "--no-deps")
-  endif()
+  # Store just _this_ project's requirements file in ${package_requirements}
+  execute_process(
+    COMMAND ${CATKIN_ENV} rosrun catkin_virtualenv collect_requirements --no-deps
+      --package-name ${PROJECT_NAME} ${lock_args}
+    OUTPUT_VARIABLE package_requirements
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
 
-  # Collect all exported pip requirements files, from this package and all dependencies
+  # Collect all of this project's inherited requirements into ${requirements_list}
   execute_process(
-    COMMAND ${CATKIN_ENV} rosrun catkin_virtualenv glob_requirements
-      --package-name ${PROJECT_NAME} ${glob_args}
+    COMMAND ${CATKIN_ENV} rosrun catkin_virtualenv collect_requirements
+      --package-name ${PROJECT_NAME} ${lock_args}
     OUTPUT_VARIABLE requirements_list
     OUTPUT_STRIP_TRAILING_WHITESPACE
   )
 
-  # Include common requirements that ROS makes available in system environment for py2
-  list(APPEND requirements_list ${catkin_virtualenv_CMAKE_DIR}/common_requirements.txt)
-
-  set(generated_requirements ${CMAKE_BINARY_DIR}/generated_requirements.txt)
-
-  # Trigger a re-configure if any requirements file changes
-  foreach(requirements_txt ${requirements_list})
-    stamp(${requirements_txt})
-    message(STATUS "Including ${requirements_txt} in bundled virtualenv")
+  # Trigger rebuild if any of the requirements files change
+  foreach(requirements_file ${requirements_list})
+    if(EXISTS ${requirements_file})
+      stamp(${requirements_file})
+    endif()
   endforeach()
 
-  # Combine requirements into one list
-  add_custom_command(OUTPUT ${generated_requirements}
-    COMMAND ${CATKIN_ENV} rosrun catkin_virtualenv combine_requirements
-      --requirements-list ${requirements_list} --output-file ${generated_requirements}
-    DEPENDS ${requirements_list}
+  add_custom_command(COMMENT "Generate virtualenv in ${CMAKE_BINARY_DIR}/${venv_dir}"
+    OUTPUT ${CMAKE_BINARY_DIR}/${venv_dir}/bin/python
+    COMMAND ${CATKIN_ENV} rosrun catkin_virtualenv venv_init ${venv_dir}
+      --python ${ARG_PYTHON_INTERPRETER} ${venv_args} --extra-pip-args ${processed_pip_args}
   )
 
-  if(${ARG_USE_SYSTEM_PACKAGES})
-    message(STATUS "Using system site packages")
-    set(venv_args "--use-system-packages")
+  if(DEFINED ARG_INPUT_REQUIREMENTS AND NOT package_requirements STREQUAL "")
+    add_custom_command(COMMENT "Lock input requirements if they don't exist"
+      OUTPUT ${package_requirements}
+      COMMAND ${CATKIN_ENV} rosrun catkin_virtualenv venv_lock ${CMAKE_BINARY_DIR}/${venv_dir}
+        --package-name ${PROJECT_NAME} --input-requirements ${ARG_INPUT_REQUIREMENTS}
+        --no-overwrite --extra-pip-args ${processed_pip_args}
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+      DEPENDS
+        ${CMAKE_BINARY_DIR}/${venv_dir}/bin/python
+        ${CMAKE_SOURCE_DIR}/${ARG_INPUT_REQUIREMENTS}
+    )
   endif()
 
-  # Generate a virtualenv, fixing up paths for devel-space
-  add_custom_command(OUTPUT ${venv_devel_dir}
-    COMMAND ${CATKIN_ENV} rosrun catkin_virtualenv build_venv
-      --root-dir ${venv_devel_dir} --requirements ${generated_requirements} --retries 3
-      --python-version ${ARG_PYTHON_VERSION} ${venv_args} --extra-pip-args ${processed_pip_args}
-    WORKING_DIRECTORY ${venv_devel_dir}/..
-    DEPENDS ${generated_requirements}
+  add_custom_command(COMMENT "Install requirements to ${CMAKE_BINARY_DIR}/${venv_dir}"
+    OUTPUT ${CMAKE_BINARY_DIR}/${venv_dir}/bin/activate
+    COMMAND ${CATKIN_ENV} rosrun catkin_virtualenv venv_install ${venv_dir}
+      --requirements ${requirements_list} --extra-pip-args ${processed_pip_args}
+    DEPENDS
+      ${CMAKE_BINARY_DIR}/${venv_dir}/bin/python
+      ${package_requirements}
+      ${requirements_list}
   )
 
-  # Generate a virtualenv, fixing up paths for install-space
-  add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/${venv_dir}
-    COMMAND ${CATKIN_ENV} rosrun catkin_virtualenv build_venv
-      --root-dir ${venv_install_dir} --requirements ${generated_requirements} --retries 3
-      --python-version ${ARG_PYTHON_VERSION} ${venv_args} --extra-pip-args ${processed_pip_args}
-    DEPENDS ${generated_requirements}
+  add_custom_command(COMMENT "Prepare relocated virtualenvs for develspace and installspace"
+    OUTPUT ${venv_devel_dir} install/${venv_dir}
+    # CMake copy_directory doesn't preserve symlinks https://gitlab.kitware.com/cmake/cmake/issues/14609
+    # COMMAND ${CMAKE_COMMAND} -E copy_directory ${venv_dir} ${venv_devel_dir}
+    # COMMAND ${CMAKE_COMMAND} -E copy_directory ${venv_dir} install/${venv_dir}
+    COMMAND mkdir -p ${venv_devel_dir} && cp -r ${venv_dir}/* ${venv_devel_dir}
+    COMMAND mkdir -p install/${venv_dir} && cp -r ${venv_dir}/* install/${venv_dir}
+
+    COMMAND ${CATKIN_ENV} rosrun catkin_virtualenv venv_relocate ${venv_devel_dir} --target-dir ${venv_devel_dir}
+    COMMAND ${CATKIN_ENV} rosrun catkin_virtualenv venv_relocate install/${venv_dir} --target-dir ${venv_install_dir}
+    DEPENDS ${CMAKE_BINARY_DIR}/${venv_dir}/bin/activate
   )
 
-  # Per-package virtualenv target
   add_custom_target(${PROJECT_NAME}_generate_virtualenv ALL
-    DEPENDS ${CMAKE_BINARY_DIR}/${venv_dir}
-    DEPENDS ${venv_devel_dir}
-    SOURCES ${requirements_list}
+    COMMENT "Per-package virtualenv target"
+    DEPENDS
+      ${venv_devel_dir}
+      install/${venv_dir}
+  )
+
+  add_custom_target(venv_lock
+    COMMENT "Manually invoked target to generate the lock file on demand"
+    COMMAND ${CATKIN_ENV} rosrun catkin_virtualenv venv_lock ${CMAKE_BINARY_DIR}/${venv_dir}
+      --package-name ${PROJECT_NAME} --input-requirements ${ARG_INPUT_REQUIREMENTS}
+      --extra-pip-args ${processed_pip_args}
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    DEPENDS
+      ${venv_devel_dir}
+      ${CMAKE_SOURCE_DIR}/${ARG_INPUT_REQUIREMENTS}
   )
 
-  install(DIRECTORY ${CMAKE_BINARY_DIR}/${venv_dir}
+  if(NOT package_requirements STREQUAL "" AND (NOT DEFINED ARG_CHECK_VENV OR ARG_CHECK_VENV))
+    file(MAKE_DIRECTORY ${CATKIN_TEST_RESULTS_DIR}/${PROJECT_NAME})
+    catkin_run_tests_target("venv_check" "${PROJECT_NAME}-requirements" "venv_check-${PROJECT_NAME}-requirements.xml"
+      COMMAND "${CATKIN_ENV} rosrun catkin_virtualenv venv_check ${venv_dir} --requirements ${package_requirements} \
+        --extra-pip-args \"${processed_pip_args}\" \
+        --xunit-output ${CATKIN_TEST_RESULTS_DIR}/${PROJECT_NAME}/venv_check-${PROJECT_NAME}-requirements.xml"
+      DEPENDENCIES ${PROJECT_NAME}_generate_virtualenv
+      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+    )
+  endif()
+
+  install(DIRECTORY ${CMAKE_BINARY_DIR}/install/${venv_dir}
     DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}
     USE_SOURCE_PERMISSIONS)
 
-  install(FILES ${generated_requirements}
-    DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION})
-
   # (pbovbel): NOSETESTS originally set by catkin here:
   # <https://github.com/ros/catkin/blob/kinetic-devel/cmake/test/nosetests.cmake#L86>
   message(STATUS "Using virtualenv to run Python nosetests: ${nosetests}")

diff --git a/cmake/common_requirements.txt b/cmake/common_requirements.txt
diff --git a/cmake/templates/program.devel.in b/cmake/templates/program.devel.in
@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-${${PROJECT_NAME}_VENV_DEVEL_DIR}/bin/python ${program_path} "$@"
+exec ${${PROJECT_NAME}_VENV_DEVEL_DIR}/bin/python ${program_path} "$@"
diff --git a/cmake/templates/program.install.in b/cmake/templates/program.install.in
@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-${${PROJECT_NAME}_VENV_INSTALL_DIR}/bin/python ${CMAKE_INSTALL_PREFIX}/${program_install_location}/${program_basename} "$@"
+exec ${${PROJECT_NAME}_VENV_INSTALL_DIR}/bin/python ${CMAKE_INSTALL_PREFIX}/${program_install_location}/${program_basename} "$@"