improved handling of test and export dependencies

industrial_ci/src/deps.py

@@ -4,6 +4,7 @@
 from catkin_pkg.packages import find_packages
 from itertools import chain
 from sys import stderr
+from copy import copy
 
 # read environment variables
 
@@ -52,53 +53,102 @@ def get_target_repo_path(): # lazy read of TARGET_REPO_PATH
 paths=dict((p.name, path) for (path,p) in known_pkgs.items())
 workspace_pkgs = set(paths.keys())
 
+recursive_deps = dict() # recursive dependencies in workspace
+
+class Dependencies(object):
+    def __init__(self, p=None):
+        if p is None:
+            self.build_deps = set()
+            self.export_deps = set()
+            self.test_deps = set()
+            self.exec_deps = set()
+            self.build_keys = set()
+            self.export_keys = set()
+            self.test_keys = set()
+        else:
+            self.build_deps = set(d.name for d in chain(p.build_depends, p.buildtool_depends)) & workspace_pkgs
+            self.export_deps = set(d.name for d in chain(p.buildtool_export_depends, p.build_export_depends)) & workspace_pkgs
+            self.test_deps = set(d.name for d in chain(p.test_depends)) & workspace_pkgs
+            self.exec_deps = set(d.name for d in chain(p.exec_depends)) & workspace_pkgs
+            self.build_keys = set(d.name for d in chain(p.build_depends, p.buildtool_depends)) # keys needed for build
+            self.export_keys = set(d.name for d in chain(p.buildtool_export_depends, p.build_export_depends)) # keys needed for transitive build
+            self.test_keys = set(d.name for d in chain(p.test_depends, p.exec_depends)) # keys needed for tests
+    def join(self, other):
+        self.build_deps |= other.build_deps | other.export_deps # export_deps turned into build_deps for dependent projects
+        self.export_deps |= other.export_deps
+        self.test_deps |= other.test_deps | other.exec_deps # exec_deps turned into test_deps for dependent projects
+        self.exec_deps |= other.exec_deps
+        self.build_keys |= other.build_keys | other.export_keys
+        self.export_keys |= other.export_keys
+        self.test_keys |= other.test_keys
+        return self
+    def getAllDeps(self):
+        return self.build_deps | self.export_deps | self.test_deps | self.exec_deps
+
 # dependencies of workspace packages within workspace
-workspace_deps = dict((p.name, set(d.name for d in chain(p.build_depends, p.buildtool_depends, p.buildtool_export_depends, p.exec_depends, p.build_export_depends, p.test_depends)) & workspace_pkgs) for p in
- known_pkgs.values())
+workspace_deps = dict((p.name, Dependencies(p)) for p in known_pkgs.values())
+
+pkgs_blacklisted |= set(p.name for p in known_pkgs.values() if p.is_metapackage()) # ignore non-target metapackages
 
-recursive_deps = dict() # recursive dependencies in workspace
 def get_recursive_deps(name):
     global recursive_deps, workspace_deps
     try:
         return recursive_deps[name]
     except KeyError:
-        deps = workspace_deps[name]
-        recursive_deps[name] = deps = reduce(lambda s,n: s.union(get_recursive_deps(n)), deps, deps) # recursive_deps = workspace_deps + recursive_workspace_deps
+        w = workspace_deps[name]
+        deps = copy(w)
+        for d in chain(w.getAllDeps()):
+            deps.join(get_recursive_deps(d))
+        recursive_deps[name] = deps
         return deps
 
 upstream_pkgs = set()
-downstream_tests = set(downstream_pkgs)
-
-# add all recursive dependencies of pre-configured dependencies
-downstream_pkgs = reduce(lambda s,n: s.union(get_recursive_deps(n)), downstream_pkgs, downstream_pkgs)
+downstream_tests = set()
+target_tests = set(t for t in target_pkgs if len(workspace_deps[t].test_keys) > 0) - tests_blacklisted
 
 for p in workspace_pkgs:
     deps = get_recursive_deps(p)
     if p in target_pkgs: # fill recursive dependencies of target packages
-        upstream_pkgs |= deps
-    if len(deps & target_pkgs) > 0: # is downstream packages
+        upstream_pkgs |= deps.build_deps
+        if p in target_tests:
+            upstream_pkgs |= deps.test_deps
+    if p in downstream_pkgs or len((deps.build_deps | deps.test_deps)  & target_pkgs) > 0: # is downstream package
         if p in pkgs_whitelisted or ( not pkgs_whitelisted and not p in pkgs_blacklisted): # match white and blacklist
-            downstream_pkgs |= deps
+            downstream_pkgs |= deps.build_deps
             downstream_pkgs.add(p)
-            downstream_tests.add(p)
+            if len(deps.test_deps  & target_pkgs) > 0 and p not in tests_blacklisted:
+                downstream_pkgs |= deps.test_deps
+                downstream_tests.add(p)
     elif p in pkgs_whitelisted: # no recursive dependency, but whitelisted
-        upstream_pkgs |= deps
+        upstream_pkgs |= deps.build_deps
         upstream_pkgs.add(p)
 
-downstream_pkgs -= upstream_pkgs # don't build upstream packages again
-downstream_pkgs -= target_pkgs # don't build target packages again
+all_pkgs = upstream_pkgs | downstream_pkgs | target_pkgs | target_tests | downstream_tests
+
+skip_keys = set()
+for a in all_pkgs:
+    r = get_recursive_deps(a)
+    skip_keys |= r.export_keys | r.test_keys # candidates for skipping
+
+for a in all_pkgs:
+    skip_keys -= get_recursive_deps(a).build_keys # don't skip build dependencies
+
+for a in target_tests | downstream_tests:
+    skip_keys -= get_recursive_deps(a).test_keys # don't skip test dependencies
+
+ignored_pkgs = set(paths.keys()) - all_pkgs
 
 def export(name, iterable):
-    print('export ' + name + '=('+' '.join(('"%s"' % i for i in iterable))+')')
+    print('export ' + name + '=('+' '.join(('"%s"' % i for i in sorted(iterable)))+')')
+    print()
 
-export('upstream_pkgs', upstream_pkgs)
+export('upstream_pkgs', upstream_pkgs - target_pkgs) # filter intra-target test dependencies
 export('target_pkgs', target_pkgs)
-export('downstream_pkgs', downstream_pkgs)
+export('downstream_pkgs', downstream_pkgs - upstream_pkgs - target_pkgs) # don't build them again
 
-target_tests = target_pkgs - tests_blacklisted
 export('target_tests', target_tests)
-export('downstream_tests', downstream_tests - target_tests - tests_blacklisted) # don't run target tests again
+export('downstream_tests', downstream_tests - target_tests) # don't run target tests again
 
-all_pkgs = upstream_pkgs | downstream_pkgs | target_pkgs
 export('rosdep_paths', (workspace_src+'/'+paths[p] for p in all_pkgs if p in paths))
-export('ignored_pkgs', set(paths.keys()) - all_pkgs)
+export('rosdep_skip_keys', (skip_keys - all_pkgs) | ignored_pkgs)
+export('ignored_pkgs', ignored_pkgs)

industrial_ci/src/tests/source_tests.sh

@@ -42,7 +42,7 @@ function try_catkin_build {
     if [ $# -gt 1 ]; then
         ici_time_start $1
         shift 1
-        if [ "$BUILDER" == catkin ]; then catkin build $OPT_VI --summarize  --no-status "$@" $CATKIN_PARALLEL_JOBS --make-args $ROS_PARALLEL_JOBS ; fi
+        if [ "$BUILDER" == catkin ]; then catkin build $OPT_VI --summarize  --no-status --no-deps "$@" $CATKIN_PARALLEL_JOBS --make-args $ROS_PARALLEL_JOBS ; fi
         ici_time_end #
     fi
 }
@@ -170,14 +170,24 @@ $ICI_SRC_PATH/deps.py > "$tmp_source"
 source "$tmp_source"
 rm $tmp_source
 
-echo "Upstream packages: ${upstream_pkgs[@]}"
-echo "Target packages: ${target_pkgs[@]}"
-echo "Downstream packages: ${downstream_pkgs[@]}"
-echo "Ignored packages: ${ignored_pkgs[@]}"
+
+cat << EOF
+
+Upstream packages: ${upstream_pkgs[@]}
+
+Target packages: ${target_pkgs[@]}
+Target tests: ${target_tests[@]}
+
+Downstream packages: ${downstream_pkgs[@]}
+Downstream tests: ${downstream_tests[@]}
+
+Ignored packages: ${ignored_pkgs[@]}
+
+EOF
 
 ici_time_start rosdep_install
 set -o pipefail # fail if rosdep install fails
-rosdep install -q --ignore-src --rosdistro $ROS_DISTRO -y --from-paths "${rosdep_paths[@]}" | { grep "executing command" || true; }
+rosdep install -q --ignore-src --rosdistro $ROS_DISTRO -y --skip-keys "${rosdep_skip_keys[*]}" --from-paths "${rosdep_paths[@]}" | { grep "executing command" || true; }
 set +o pipefail
 ici_time_end  # rosdep_install